[ Oct 18 17:54:07 Enabled. ]
[ Oct 18 17:54:07 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Oct 18 17:54:07 Method "start" exited with status 0. ]
[ Oct 18 17:54:07 Rereading configuration. ]
[ Oct 18 17:54:07 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.132523314Z","hostname":"buskin","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.132688658Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.185189924Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.185228445Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.296643133Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.305254529Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.312734239Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:07.914727452Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.051327864Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.051376785Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.051503428Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.05157876Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.05159957Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.05160788Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054766202Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:08.054772422Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054783142Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054786782Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054799133Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054864774Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054932316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054954876Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.054962717Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.06476159Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.06478185Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.160998812Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.538830669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.5388642Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.538869Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.53888467Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.53889637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.538916071Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.538921011Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.547017495Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.547040756Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"62298c00-fb90-4e92-a4b4-95bca961978c (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:08.629960884Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027546881Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027598452Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027628363Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027635173Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027645943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027654034Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027661294Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027666694Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027683214Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027688424Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027693004Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027707955Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027713025Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027716965Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027734805Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027739685Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027744125Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027749316Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027753516Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027765556Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027771736Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027798237Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027803137Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027808517Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027812347Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027816197Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027836208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027842068Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027846808Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027851308Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027854248Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027857568Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027879179Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027883459Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027889839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027893019Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027896389Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027900739Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02792629Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02793113Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02793493Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02793842Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02794156Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02794599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.02794991Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027983051Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027987241Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027991511Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027995051Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.027999331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028002921Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028007182Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028032192Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028040832Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028044242Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028055172Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028058723Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028061883Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028086503Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028090553Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028093873Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028123934Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028134814Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028143354Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028160125Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028164585Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028168505Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028193776Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.028198076Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.03892056Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:739"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.038965551Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999923629s","next_cleanup":"Instant { tv_sec: 1786, tv_nsec: 706811491 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.038983932Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.038994692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.039041443Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.041515699Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.041879158Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.043563996Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.1.164,\n                last: 10.151.1.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.1.164,\n        10.151.1.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV2 {\n        rack_subnet: Ipv6Net {\n            addr: fd00:1122:3344:100::,\n            width: 56,\n        },\n        infra_ip_first: 10.151.1.174,\n        infra_ip_last: 10.151.1.174,\n        ports: [\n            PortConfigV2 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Net {\n                                addr: 0.0.0.0,\n                                width: 0,\n                            },\n                        ),\n                        nexthop: 10.151.1.100,\n                        vlan_id: None,\n                        rib_priority: None,\n                    },\n                ],\n                addresses: [\n                    UplinkAddressConfig {\n                        address: V4(\n                            Ipv4Net {\n                                addr: 10.151.1.174,\n                                width: 24,\n                            },\n                        ),\n                        vlan_id: None,\n                    },\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n                lldp: None,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044042327Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1073"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044049227Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044125239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.04418815Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.04420606Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044216001Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1136"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044258492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044273792Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044285272Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044290233Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1178"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044294993Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044304383Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044341974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044380775Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044733203Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage: Plan {\n    rack_id: 33bc13e3-6beb-4d1b-8b04-6e3da281e5d7,\n    sleds: {\n        [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n            generation: 0,\n            schema_version: 1,\n            body: StartSledAgentRequestBody {\n                id: 08066c3d-b7ed-49d2-8578-f8769fb94736 (sled),\n                rack_id: 33bc13e3-6beb-4d1b-8b04-6e3da281e5d7,\n                use_trust_quorum: false,\n                is_lrtq_learner: false,\n                subnet: Ipv6Subnet {\n                    net: Ipv6Net {\n                        addr: fd00:1122:3344:101::,\n                        width: 64,\n                    },\n                },\n            },\n        },\n    },\n    config: RackInitializeRequest {\n        trust_quorum_peers: None,\n        bootstrap_discovery: OnlyOurs,\n        ntp_servers: [\n            \"0.pool.ntp.org\",\n        ],\n        dns_servers: [\n            1.1.1.1,\n            9.9.9.9,\n        ],\n        internal_services_ip_pool_ranges: [\n            V4(\n                Ipv4Range {\n                    first: 10.151.1.164,\n                    last: 10.151.1.173,\n                },\n            ),\n        ],\n        external_dns_ips: [\n            10.151.1.164,\n            10.151.1.165,\n        ],\n        external_dns_zone_name: \"oxide.test\",\n        external_certificates: \"<redacted>\",\n        recovery_silo: RecoverySiloConfig {\n            silo_name: Name(\n                \"recovery\",\n            ),\n            user_name: UserId(\n                \"recovery\",\n            ),\n            user_password_hash: NewPasswordHash(\n                PasswordHashString {\n                    string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                    encoding: B64,\n                },\n            ),\n        },\n        rack_network_config: RackNetworkConfigV2 {\n            rack_subnet: Ipv6Net {\n                addr: fd00:1122:3344:100::,\n                width: 56,\n            },\n            infra_ip_first: 10.151.1.174,\n            infra_ip_last: 10.151.1.174,\n            ports: [\n                PortConfigV2 {\n                    routes: [\n                        RouteConfig {\n                            destination: V4(\n                                Ipv4Net {\n                                    addr: 0.0.0.0,\n                                    width: 0,\n                                },\n                            ),\n                            nexthop: 10.151.1.100,\n                            vlan_id: None,\n                            rib_priority: None,\n                        },\n                    ],\n                    addresses: [\n                        UplinkAddressConfig {\n                            address: V4(\n                                Ipv4Net {\n                                    addr: 10.151.1.174,\n                                    width: 24,\n                                },\n                            ),\n                            vlan_id: None,\n                        },\n                    ],\n                    switch: Switch0,\n                    port: \"qsfp0\",\n                    uplink_port_speed: Speed40G,\n                    uplink_port_fec: None,\n                    bgp_peers: [],\n                    autoneg: false,\n                    lldp: None,\n                },\n            ],\n            bgp: [],\n            bfd: [],\n        },\n        allowed_source_ips: Any,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044974778Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.044979768Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.045013479Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.045022049Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.045181593Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.045347917Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.045435648Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 08066c3d-b7ed-49d2-8578-f8769fb94736 (sled), rack_id: 33bc13e3-6beb-4d1b-8b04-6e3da281e5d7, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.046175495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221"}
{"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.046231477Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4603"}
{"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.050374251Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"}
{"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.056153873Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"}
{"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.056284696Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"}
{"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060363708Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"}
{"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.06042075Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060520542Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:43269","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 08066c3d-b7ed-49d2-8578-f8769fb94736 (sled), rack_id: 33bc13e3-6beb-4d1b-8b04-6e3da281e5d7, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060643915Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060656395Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060686966Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/server.rs:44"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060698756Z","hostname":"buskin","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060705276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060710667Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060767448Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060774138Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060803018Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060819259Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060824419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.060831439Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.0608715Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.06087983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.076850784Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.097181137Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.11441117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.13681034Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:391"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.136899262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.136913662Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:403"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.136926143Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 134.159979ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.140431532Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0fa141d6-4659-4da9-a1aa-cf3a1c4b5928\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0fa141d6-4659-4da9-a1aa-cf3a1c4b5928\" }"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_e40afb95-7af8-4f62-aab4-e63b3bd02855/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.150891411Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_e40afb95-7af8-4f62-aab4-e63b3bd02855/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.175129543Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_e40afb95-7af8-4f62-aab4-e63b3bd02855/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.175173374Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.180781022Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:418"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.180794272Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"}
{"msg":"Ensuring dataset oxi_e40afb95-7af8-4f62-aab4-e63b3bd02855/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.180799392Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.266000193Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.27510649Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 68.021879ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.275808956Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e614c31-b51e-4af6-b762-8b1ccc0b1e0f\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e614c31-b51e-4af6-b762-8b1ccc0b1e0f\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.34490208Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 85.559323ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.345383971Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c9ad2eb-f720-4c13-820b-b1915f37c927\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c9ad2eb-f720-4c13-820b-b1915f37c927\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.431546314Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 211.997774ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.431968704Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df9f8ff3-38a4-4f61-a7cc-e7cf72e315e0\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df9f8ff3-38a4-4f61-a7cc-e7cf72e315e0\" }"}
{"msg":"Mounting oxi_e40afb95-7af8-4f62-aab4-e63b3bd02855/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.485679107Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.498864667Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.498964239Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.49898168Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.499028101Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.646079551Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 148.571206ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.646964051Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4933168-46cb-454b-9933-542f70f052f6\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d4933168-46cb-454b-9933-542f70f052f6\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.796973228Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 308.691644ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:09.797676754Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f310fee-31df-4d00-8b59-80424d1d6659\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f310fee-31df-4d00-8b59-80424d1d6659\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.107937651Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 405.034983ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.108646817Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"450434da-4d73-4bcc-8ca9-c95b060ffdee\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"450434da-4d73-4bcc-8ca9-c95b060ffdee\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.220471934Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.22599502Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.226035071Z","hostname":"buskin","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.226096463Z","hostname":"buskin","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:65402","producer_id":"08066c3d-b7ed-49d2-8578-f8769fb94736"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:10.22728161Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"211.41666ms"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.227961755Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.227968235Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.228047107Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:923","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.228052197Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:10.439629087Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"561.000161ms"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:10.514578264Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:11.002193801Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"924.614722ms"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:11.927944189Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.192787676s"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.641471423Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2679"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.641507864Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2897"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.641513114Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2549"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.641517524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2596"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.641521734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2978"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.641525654Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3020"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.643445128Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2822"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.644727087Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"}
{"msg":"Profile for oxz_switch:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgs\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"id\" value='e2c94a14-1106-4e76-8538-bace2c8fb163'/>\n        <propval type=\"astring\" name=\"rack_id\" value='33bc13e3-6beb-4d1b-8b04-6e3da281e5d7'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/wicketd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"address\" value='[::1]:12226'/>\n        <propval type=\"astring\" name=\"artifact-address\" value='[fdb0:18c0:4d0d:9fb2::2]:12227'/>\n        <propval type=\"astring\" name=\"baseboard-file\" value='/opt/oxide/baseboard.json'/>\n        <propval type=\"astring\" name=\"mgs-address\" value='[::1]:12225'/>\n        <propval type=\"astring\" name=\"nexus-proxy-address\" value='[::]:12229'/>\n        <propval type=\"astring\" name=\"rack-subnet\" value='fd00:1122:3344::'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/switch_zone_setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"gz_local_link_addr\" value='fe80::8:20ff:fe07:3200'/>\n        <property type=\"astring\" name=\"link_local_links\">\n          <astring_list>\n            <value_node value=\"tfportrear0_0\" />\n            <value_node value=\"oxBootstrap0\" />\n          </astring_list>\n        </property>\n        <propval type=\"astring\" name=\"bootstrap_addr\" value='fdb0:18c0:4d0d:9fb2::2'/>\n        <propval type=\"astring\" name=\"bootstrap_vnic\" value='oxBootstrap0'/>\n        <propval type=\"astring\" name=\"baseboard_info\" value='{\"type\":\"pc\",\"identifier\":\"buskin\",\"model\":\"i86pc\"}'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/dendrite\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_id\" value='08066c3d-b7ed-49d2-8578-f8769fb94736'/>\n        <propval type=\"astring\" name=\"rack_id\" value='33bc13e3-6beb-4d1b-8b04-6e3da281e5d7'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12224'/>\n        <propval type=\"astring\" name=\"front_ports\" value='1'/>\n        <propval type=\"astring\" name=\"rear_ports\" value='1'/>\n        <propval type=\"astring\" name=\"port_config\" value='/opt/oxide/dendrite/misc/softnpu_single_sled_config.toml'/>\n        <propval type=\"astring\" name=\"mgmt\" value='uds'/>\n        <propval type=\"astring\" name=\"uds_path\" value='/opt/softnpu/stuff'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/tfport\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"host\" value='[::1]'/>\n        <propval type=\"astring\" name=\"port\" value='12224'/>\n        <propval type=\"astring\" name=\"flags\" value='--sync-only'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/lldpd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"board_rev\" value='softnpu_front_1_rear_1'/>\n        <propval type=\"astring\" name=\"scrimlet_id\" value='buskin'/>\n        <propval type=\"astring\" name=\"scrimlet_model\" value='i86pc'/>\n        <propval type=\"astring\" name=\"address\" value='[::1]:12230'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/pumpkind\">\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mgd\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"sled_uuid\" value='08066c3d-b7ed-49d2-8578-f8769fb94736'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='33bc13e3-6beb-4d1b-8b04-6e3da281e5d7'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/mg-ddm\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"mode\" value='transit'/>\n        <propval type=\"astring\" name=\"dendrite\" value='true'/>\n        <propval type=\"astring\" name=\"sled_uuid\" value='08066c3d-b7ed-49d2-8578-f8769fb94736'/>\n        <propval type=\"astring\" name=\"rack_uuid\" value='33bc13e3-6beb-4d1b-8b04-6e3da281e5d7'/>\n        <propval type=\"astring\" name=\"interfaces\" value='tfportrear0_0/ll'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/uplink\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.644885531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:12.645052314Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 34.173026ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:12.895621002Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 50.871401ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:13.092565648Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 150.07291ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:13.269777715Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 480.004915ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:13.443821579Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 759.173194ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:13.984612378Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:14.122017098Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.556843169s"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 581.431101ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:14.754883374Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.274523035s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:15.417955318Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.292749205s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:16.833509563Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.563979491Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564053353Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564061153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564069973Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564239587Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564424641Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564467142Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564478033Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.564531024Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:43269","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807467098Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.80758462Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807704023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807751354Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807759955Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807824106Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807848697Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.807857597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:18.809013833Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.193411734s"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.813167358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4245","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.813182308Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4261"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:18.813187268Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.075153255Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:38982"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.075531254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.075544834Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.075628926Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.075656647Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.075749989Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"0e150506-f281-42fb-89a4-512ec3728017","remote_addr":"[fd00:1122:3344:101::1]:38982","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":387,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.10697102Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:45738"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.107091943Z","hostname":"buskin","pid":1572,"uri":"/sled-role","method":"GET","req_id":"0a3c46f8-483d-4327-a188-76b6a05b3799","remote_addr":"[fd00:1122:3344:101::1]:45738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.108494835Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.108530686Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.108817622Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.109120339Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:882"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 20335afa-6eea-4f23-a3f5-8df78c6c8e50, pool_id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: c26a1128-eca4-432a-a9e7-cab8cc50a3cb, pool_id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: ee2d4b24-e1e6-4689-8de5-2a74c881666f, pool_id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 1838a00e-8d64-49d4-8b47-216dfa7830a8, pool_id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 37b21758-d6b3-4a32-946c-b6c04b417dd7, pool_id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: f6b6c54a-2880-47d3-9451-12d39430d203, pool_id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 5d4a1bc6-9204-44c2-8b9e-80a1dd662d32, pool_id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 74d5a544-0c0e-49bf-8cff-fb634583f926, pool_id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 4e7cff97-1fa7-4eba-8866-b612f9be3bc3, pool_id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138083059Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138160541Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60647"}
{"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.13857684Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 20335afa-6eea-4f23-a3f5-8df78c6c8e50, pool_id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: c26a1128-eca4-432a-a9e7-cab8cc50a3cb, pool_id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: ee2d4b24-e1e6-4689-8de5-2a74c881666f, pool_id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 1838a00e-8d64-49d4-8b47-216dfa7830a8, pool_id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 37b21758-d6b3-4a32-946c-b6c04b417dd7, pool_id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: f6b6c54a-2880-47d3-9451-12d39430d203, pool_id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 5d4a1bc6-9204-44c2-8b9e-80a1dd662d32, pool_id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 74d5a544-0c0e-49bf-8cff-fb634583f926, pool_id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 4e7cff97-1fa7-4eba-8866-b612f9be3bc3, pool_id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138625971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138638701Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138650422Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138659982Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138696583Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138705993Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138711773Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.138716183Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.295978756Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:19.296009466Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8853a1c1-d102-4f80-9012-356473e594ff (zpool)"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/08066c3d-b7ed-49d2-8578-f8769fb94736)","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:19.566970129Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:20.261028168Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"291083675488840079"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:20.26110375Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7620151230303063215"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.488354687Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.500743679Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.50824358Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.508309441Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.508320452Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.508328792Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.508336712Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.508431794Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.508439824Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.54246751Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_8853a1c1-d102-4f80-9012-356473e594ff/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.549876278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/08066c3d-b7ed-49d2-8578-f8769fb94736)","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:20.57016328Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Looking for unencrypted datasets in oxp_8853a1c1-d102-4f80-9012-356473e594ff","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.716902663Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.725242853Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.725302744Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.725313655Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.725319725Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.725326445Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.734601246Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.734637837Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.827185445Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.8353165Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.835350681Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.835360011Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.835389042Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.835408032Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.835529505Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.835562696Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.870451911Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_8523f2c5-b045-4f20-8f22-89f361c6e33c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:20.878353861Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"Looking for unencrypted datasets in oxp_8523f2c5-b045-4f20-8f22-89f361c6e33c","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.056331644Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.064642124Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.064680835Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.064689845Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.064696555Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.064707385Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.073631278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.073661469Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"24197587-83af-46c0-a36e-5e4a26350d21 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.162098394Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.168789736Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.168834757Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.168843938Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.168857398Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.168872188Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.168995401Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.169009871Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.203000776Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_24197587-83af-46c0-a36e-5e4a26350d21/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.211337195Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"Looking for unencrypted datasets in oxp_24197587-83af-46c0-a36e-5e4a26350d21","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.380901058Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.38891146Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.388950051Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.388976602Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.388985952Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.389015763Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.398427607Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.398455078Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.493216796Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501112996Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501170858Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501181318Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501216249Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501227109Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501363392Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.501372872Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.536663106Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_956d8649-5e02-4c83-a766-709d6c1c19f4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.544066845Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/08066c3d-b7ed-49d2-8578-f8769fb94736)","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:21.572294428Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Looking for unencrypted datasets in oxp_956d8649-5e02-4c83-a766-709d6c1c19f4","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.708439929Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.715465239Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.71551543Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.71552232Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.715530091Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.715536851Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.724777371Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.724811332Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.816177563Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.823965511Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.824015342Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.824040192Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.824047623Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.824124054Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.824272898Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.824282558Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.860721448Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_4e3d3772-4d1a-4cd4-9451-d3f5402e0376/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:21.869420366Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"Looking for unencrypted datasets in oxp_4e3d3772-4d1a-4cd4-9451-d3f5402e0376","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.043590433Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.051302569Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.05135104Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.051361481Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.051367601Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.051372921Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.060081219Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.06011334Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"aa3e117f-01a2-4173-8076-71ed9651de49 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.152523295Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160349483Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160373884Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160381734Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160401684Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160412285Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160521607Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.160528317Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.193701183Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_aa3e117f-01a2-4173-8076-71ed9651de49/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.200921197Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"Looking for unencrypted datasets in oxp_aa3e117f-01a2-4173-8076-71ed9651de49","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.376537598Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.3849749Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.385033671Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.385040751Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.385046672Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.385051672Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.394576669Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.394607299Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"e09cb849-5302-41dd-87a5-36e8723c48fb (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.493552153Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.50131957Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.501358341Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.501365401Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.501373361Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.501384312Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.501486104Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.501494254Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.537367221Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_e09cb849-5302-41dd-87a5-36e8723c48fb/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.546293355Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/08066c3d-b7ed-49d2-8578-f8769fb94736)","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:22.574502347Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Looking for unencrypted datasets in oxp_e09cb849-5302-41dd-87a5-36e8723c48fb","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.717508295Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.725422145Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.725461336Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.725472086Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.725480566Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.725487197Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.735047974Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.735067495Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.830909658Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838560162Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838597713Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838614474Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838655424Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838665865Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838808488Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.838816548Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.875891483Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_c6223f26-38f9-4483-9d17-efd8e60dd490/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:22.885227795Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"Looking for unencrypted datasets in oxp_c6223f26-38f9-4483-9d17-efd8e60dd490","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.058326428Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.066458563Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.066485194Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.066500464Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.066506354Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.066535145Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.076679456Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.076703497Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.187665194Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.196963146Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.196998887Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.197008287Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.197033818Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.197042508Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.197258243Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.197270383Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.23446908Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.243294132Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"}
{"msg":"Looking for unencrypted datasets in oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.420935398Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.428995182Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429062663Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429131435Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429318499Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e40afb95-7af8-4f62-aab4-e63b3bd02855 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62298c00-fb90-4e92-a4b4-95bca961978c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429398111Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429406751Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429479983Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429687578Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:868"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429717298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429726858Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:894","generation":"Generation(1)"}
{"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429734388Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"}
{"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429738379Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429765779Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.42977478Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:910"}
{"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429819181Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:798","gen":"Generation(1)"}
{"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429825821Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:915"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.429897652Z","hostname":"buskin","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"fb1884bc-ca5a-42b8-b0dd-14de6f052ddf","remote_addr":"[fd00:1122:3344:101::1]:60647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4291616,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.461102883Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.461267447Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:35122"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.461483632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.461489262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.512886823Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.512933424Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.564625791Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.564663372Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/08066c3d-b7ed-49d2-8578-f8769fb94736)","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:23.576968562Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.616116874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"No ledger in /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.616160205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.616166875Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.616230887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.661648641Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.661690792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.661696632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.70987281Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.713619855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.71732496Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.729762473Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.73401418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:23.738080772Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:24.005019413Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"21.451816255s"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/08066c3d-b7ed-49d2-8578-f8769fb94736)","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:24.579147691Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"}
{"msg":"Profile for oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService1'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:2::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:2::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:2::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:2::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:25.318172275Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:25.318244277Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"}
{"msg":"Booting oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:25.318433181Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService2'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:1::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:1::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:1::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:1::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:25.801867654Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:26.281472298Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"}
{"msg":"Profile for oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService3'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:3::2'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:3::1'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:3::1]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='[fd00:1122:3344:3::1]:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:26.28156186Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:26.281614821Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 26.295178ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.289803088Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:26.289849479Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:26.293960312Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 142.471688ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.327240101Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 64.950725ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.412332759Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 69.929629ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.417682181Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 195.479ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.479653093Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 108.278131ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.512188523Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 75.939184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.578023263Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 104.617399ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.734077598Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 276.739806ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.758069215Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 275.385509ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.778354777Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 207.928374ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:26.883297517Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 1.126623595s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:27.045938742Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 291.508488ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:27.063660965Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 736.265306ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:27.103178346Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 1.054917621s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:27.364460997Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 1.395360648s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:27.849501226Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 1.213849084s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:28.182928431Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 825.132846ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:28.429702783Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:28.567019741Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12885496655507240414"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:28.567137793Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6974983097420402160"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:28.952694166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4269"}
{"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:28.952754327Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4279","gateway":"fd00:1122:3344:101::1"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:28.958629201Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4358"}
{"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.061868363Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4406"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.061923304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433"}
{"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.099691154Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"}
{"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.099729355Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4310"}
{"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.151606367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4352"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.151643038Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4413","rack_subnet":"fd00:1122:3344::"}
{"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.170806144Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4424"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.170837945Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4466"}
{"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.238976817Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4498"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.239017928Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4504"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 1.402889407s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:29.254390058Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 1.220221456s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:29.264711194Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.342019884Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.342076896Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.353903865Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:53630"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.354163951Z","hostname":"buskin","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"dd3d1373-98bc-4650-9532-1f08ed96ecce","remote_addr":"[fd00:1122:3344:101::2]:53630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"200"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.374999336Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 1.305274493s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:29.40502391Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.410112355Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.410218468Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.410620627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"0"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.410629847Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.1.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.448403208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495593383Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495659174Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495670015Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495675195Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495692115Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495700605Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495708505Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495716316Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495722826Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495729256Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495737256Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495745536Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495751846Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495756926Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495761857Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495766297Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495781357Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495786027Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495790557Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495795157Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495799958Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495804767Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:29.495809298Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:30.264025417Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"291083675488840079"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:30.2641395Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7620151230303063215"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 579.381508ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:30.493241238Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb\") failed: Property not found. retry in 1.38593567s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:30.667070578Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88\") failed: Property not found. retry in 1.000925378s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:30.719288457Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6\") failed: Property not found. retry in 606.64242ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:31.083682648Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.069720749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_1fb67d4e-bac4-4e8f-b10a-4ef050bccb88"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.06979169Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_5a24845a-5b82-4ab6-b0ce-11d63d68b9cb"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.069804701Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_7a37cb7b-e967-4955-ad9c-4441ce0d32e6"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.069821761Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.070167309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.070509047Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"74e2ba67-328e-49b3-ac51-0a573edda02a","remote_addr":"[fd00:1122:3344:101::1]:35122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8609164,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.072493612Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.118222583Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.170891063Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.216670426Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.216680686Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.246907045Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.247666262Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.247690883Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.277040991Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.277134583Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:50903"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.277347268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.277357239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.294927259Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.29497353Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.312275364Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.312319215Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.328761659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.328817071Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.328982204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.329060316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.340070477Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.371734438Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.372351792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.372357463Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.40166862Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 176, 131]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.401701871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"boundary_ntp"}
{"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.402146371Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.402156481Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.402480939Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.402870058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.424703525Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:32.441866946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_64822dce-d64a-403f-bf53-d84635424320:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService4'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value='fd00:1122:3344:100::/56'/>\n        <propval type=\"boolean\" name=\"boundary\" value='true'/>\n        <propval type=\"astring\" name=\"boundary_pool\" value='boundary-ntp.control-plane.oxide.internal'/>\n        <propval type=\"astring\" name=\"server\" value='0.pool.ntp.org'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte0'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.3.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.3.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:33.07246705Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_ntp_64822dce-d64a-403f-bf53-d84635424320 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:33.072640884Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 35.093971ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:33.200278371Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 131.479404ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:33.24280686Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 105.055617ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:33.399934019Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 413.175276ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:33.517994929Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 941.434568ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:33.939716865Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 850.107974ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:34.892081949Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 1.056227109s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:35.752253343Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320\") failed: Property not found. retry in 1.353024173s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:36.817872957Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.190581576Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_ntp_64822dce-d64a-403f-bf53-d84635424320"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.190664078Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.190962105Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.191303932Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"af767690-0c77-4765-ba85-f307c83670bf","remote_addr":"[fd00:1122:3344:101::1]:50903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5914102,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.193796619Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.224468348Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.224613211Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:44038"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_64822dce-d64a-403f-bf53-d84635424320': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:38.233513084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.233642767Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"5a503c60-fcfa-4c28-b92c-b526987dea5e","remote_addr":"[fd00:1122:3344:101::1]:44038","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8872,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:38.236192355Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.362620915Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:38.362792689Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:59081"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_64822dce-d64a-403f-bf53-d84635424320': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.371492703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:42.371612816Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"bdcd1c3f-ec0c-4fd0-a040-ec4b5e431c92","remote_addr":"[fd00:1122:3344:101::1]:59081","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008648,"response_code":"500"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.37357232Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"4809265991385465253"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.373734684Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3909211028453507039"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.373890237Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11979277470628982865"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.374041711Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12885496655507240414"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.374181344Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6974983097420402160"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.374315037Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"291083675488840079"}
{"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.3744543Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7620151230303063215"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:42.375015223Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:42.532060451Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:42.532236155Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:33386"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_64822dce-d64a-403f-bf53-d84635424320': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:42.540697867Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:42.54081597Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"e453b4fe-88d8-4608-b09f-59af067e3a44","remote_addr":"[fd00:1122:3344:101::1]:33386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8493,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:42.543836079Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:42.688357111Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:42.688503324Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:49074"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_64822dce-d64a-403f-bf53-d84635424320': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-18T17:54:46.002342544Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.002407895Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"fb3648cc-9b9e-415b-b9af-d47eafd27b46","remote_addr":"[fd00:1122:3344:101::1]:49074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007865,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.003998851Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.28052385Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.280668333Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39824"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.301212581Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"533a8015-407e-42dd-bd0f-0fde8922b732","remote_addr":"[fd00:1122:3344:101::1]:39824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":20409,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1729274085.3730578, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.304839774Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.304852074Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1298"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.335542553Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.335660396Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:38201"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.336072805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.336083195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.355491598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.355544629Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.37317246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.373223092Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.391209001Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.391250272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.439746147Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.439789298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.491822863Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.491858984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.544264828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.544307229Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.592862015Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.592906256Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.643396246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.643440817Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.64358858Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.643698613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.674156167Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.674199708Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.674209398Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.674218928Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.674228058Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.674330931Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c24ab5f5-b36b-4dee-9f53-60c5bfa29661","remote_addr":"[fd00:1122:3344:101::1]:38201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\", TimeNotSynchronized), (\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\", TimeNotSynchronized), (\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\", TimeNotSynchronized), (\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\", TimeNotSynchronized), (\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\", TimeNotSynchronized)]","latency_us":338478,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.674515075Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"137.453533ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c24ab5f5-b36b-4dee-9f53-60c5bfa29661\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c24ab5f5-b36b-4dee-9f53-60c5bfa29661\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.814063204Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.814442672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.814462983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.832136475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.832168316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.84903511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.849056331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.866451847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.866481058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.883538236Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.883561487Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.899657153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.899688554Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.916092858Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.916122879Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.932514612Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.932536212Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.949169841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.949198982Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.949374596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.949523109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.978586671Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.978627672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.978637532Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.978645443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.978654883Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:46.978758515Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"53ef5d20-b5d8-4530-9f39-9368ae94d807","remote_addr":"[fd00:1122:3344:101::1]:38201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\", TimeNotSynchronized), (\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\", TimeNotSynchronized), (\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\", TimeNotSynchronized), (\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\", TimeNotSynchronized), (\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\", TimeNotSynchronized)]","latency_us":164548,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:46.978908829Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"98.125762ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53ef5d20-b5d8-4530-9f39-9368ae94d807\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:54:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53ef5d20-b5d8-4530-9f39-9368ae94d807\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.079091571Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.079454549Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.079470689Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.013899907Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.013938208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.031797365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.031834695Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.048953756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.048991526Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.066900274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.066935255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.084494565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.084535736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.102466744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.102504195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.121234412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.121275263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.138535246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.138584337Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.138771731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.138933985Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.219246824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.219310746Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.219317326Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.219325966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.219331806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.308731233Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.312592551Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.316433458Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.320321577Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.32442926Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.329289611Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.333496397Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.337859386Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.342228816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:47.346550474Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService9'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::4'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::4]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='8e79611e-ba88-4a64-8bf6-28d7df64e66a'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::4]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::4]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.356650915Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService6'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::5]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='69bfa368-1d69-4d23-88e9-3383e573f0eb'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::5]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::5]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.35683001Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.356837173Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.362052648Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService7'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::7'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::7]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='2d1c58db-a735-48db-90ac-891756bdde5d'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::7]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::7]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.374107182Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService5'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::3'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::3]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='03d0126b-4566-466f-bb9c-de9896115c34'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::3]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::3]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.374310606Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.374330753Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.379487778Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService8'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='[fd00:1122:3344:101::6]:32221'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroach-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"zone_id\" value='68422c74-fc22-434f-add8-652fc427fc03'/>\n        <propval type=\"astring\" name=\"cockroach_address\" value='[fd00:1122:3344:101::6]:32221'/>\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::6]:32222'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.384130257Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:50.38423704Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 49.301977ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.505605199Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 38.733129ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.513476222Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 42.748201ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.5196501Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 72.430008ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.526096921Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 46.031489ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.532229342Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 59.188762ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.560212787Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 114.465556ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.617184977Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 119.558499ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.666574847Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 247.651193ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.706967905Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 110.83957ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.730521234Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 78.32511ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:50.969826395Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 243.629208ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:51.25713436Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 114.209489ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:51.489583455Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 443.78318ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:51.527891762Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 214.568604ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:51.555911817Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 210.278538ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:51.832109895Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 492.942322ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.039725094Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 570.977929ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.193480917Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 980.113588ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.302575131Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 553.802519ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.403383437Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 579.724812ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.885376224Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 738.398804ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.896795085Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 635.75826ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.908708517Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 1.057857144s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:52.98041289Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 843.993059ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:53.448075597Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 1.079107311s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:53.481601559Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.36663636s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:53.557295549Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 1.186310675s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:53.64811467Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 1.13090862s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:54.051870588Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 534.384077ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:54.304885532Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 694.246759ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:54.574040555Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 1.431726173s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:54.847846872Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 712.404074ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:54.85635329Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 775.971177ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:54.936419393Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 1.246628232s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:55.195793628Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 952.995578ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:55.279056092Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 1.482880995s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:55.580226193Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.254498093s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:55.724418993Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 651.825913ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:56.244659642Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 1.451862612s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:56.292160105Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 1.294208726s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:56.45516286Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 618.98188ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:56.911636127Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 976.389981ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:56.993789316Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 1.217010827s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:57.080863978Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 785.704912ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:57.546630712Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 1.191093613s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:57.757360139Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 1.483756319s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:57.764866294Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 549.280643ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:57.98045016Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 1.048211776s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:58.309926354Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 899.924064ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:58.34323223Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:58.407539771Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52904"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:54:58.407749411Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"97d0c979-d9c8-44bb-aaa0-776fd406b01f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52904","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 636.409353ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:58.53969575Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 891.624489ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:58.959033846Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.185864855s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.187118394Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 625.398321ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.256470952Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 922.756867ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.264808802Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 973.485384ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.37016651Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.746435751Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"259.190851ms"}
{"msg":"wait for service Some(\"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a\") failed: Property not found. retry in 805.004322ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.862318312Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03\") failed: Property not found. retry in 1.077882805s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:54:59.892728239Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d\") failed: Property not found. retry in 550.321929ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:00.200173615Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb\") failed: Property not found. retry in 624.659251ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:00.355597883Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.044973285s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:00.38673234Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 503.461516ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:01.445661817Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 885.483833ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:01.96075117Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 974.183908ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:02.860349851Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 656.834994ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:04.034693648Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.087386371s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:04.720574343Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 870.83681ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:05.825553399Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.371036044s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:06.708922406Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 624.177084ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:08.097157061Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 917.36785ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:08.73155235Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 588.229672ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:09.661320256Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 779.095183ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:10.260835633Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 539.821614ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:11.051736659Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 930.596343ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:11.602507166Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 1.219624207s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:12.545373784Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34\") failed: Property not found. retry in 755.250876ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:13.777391243Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.557207336Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.557268396Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_69bfa368-1d69-4d23-88e9-3383e573f0eb"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.557274452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_2d1c58db-a735-48db-90ac-891756bdde5d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.557279749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_68422c74-fc22-434f-add8-652fc427fc03"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.557284868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_8e79611e-ba88-4a64-8bf6-28d7df64e66a"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.557329438Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.571501257Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.585418044Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4b95b950-eb0f-4faf-b20e-6e396ba3929b","remote_addr":"[fd00:1122:3344:101::1]:38201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":27519125,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.617539299Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:55162"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.651463949Z","hostname":"buskin","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"5dfd9717-9bf1-47b3-9a1d-7e11cba304a7","remote_addr":"[fd00:1122:3344:101::1]:55162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService5/omicron6 in oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03d0126b-4566-466f-bb9c-de9896115c34 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService5/omicron6] executed and failed with status: exit status: 1  stdout: \n  stderr: ipadm: Could not get address: Object not found","latency_us":33682,"response_code":"500"}
{"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:14.651664588Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1027","retry_after":"145.186178ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5dfd9717-9bf1-47b3-9a1d-7e11cba304a7\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5dfd9717-9bf1-47b3-9a1d-7e11cba304a7\" }"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:14.824917933Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3720"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:19.506205689Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3738"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.555677866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.555779147Z","hostname":"buskin","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"a0276493-eec0-4875-91f9-0ef2d845a498","remote_addr":"[fd00:1122:3344:101::1]:55162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":13720249,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.556213614Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63276"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.556661177Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"19511586-e7e5-4203-9795-9062bc1fca21","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63276","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:28.559935805Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"595.957572ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.593205257Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.593330754Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34297"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.593653777Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.593665796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.618701921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.618746594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.639588772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.639623319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.660468133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.660504059Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.67934924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.67938451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.698807023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.698838712Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.717055861Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.717082984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.735658533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.735688918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.754762785Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.754806943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.881808766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.88185238Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.973592591Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:28.973633289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.089170532Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.089208192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.195064213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.19510927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.291453687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.291519286Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.371987611Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.372026105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.462245979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.462288074Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.553552643Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.553660682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.641295911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.641339309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.729997225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.730116257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.801411678Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.801450182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.885421451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.885539471Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.885774826Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.885899807Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989012404Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989067652Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989073708Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989079212Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989084818Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989109643Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989114865Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989119788Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989124672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989128225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989133035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989136354Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989141135Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989144425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989150547Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989153839Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989159351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989166955Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.989170339Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:29.989228024Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"32e85499-da82-40a7-b25b-b9d1d5ae0f86","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized)]","latency_us":1392022,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:29.98941327Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"104.844746ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32e85499-da82-40a7-b25b-b9d1d5ae0f86\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"32e85499-da82-40a7-b25b-b9d1d5ae0f86\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.095992185Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.096386103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.096392703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.115518642Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.115555618Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.134624738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.134661133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.153661021Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.153692108Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.171071651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.171105458Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.189719957Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.189753521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.20705978Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.207091309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.224480373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.224513261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.242324913Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.242357716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.261100075Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.261140351Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.280489508Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.280523878Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.29926063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.299291202Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.315704717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.315738937Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.331475684Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.331508976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.349278927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.34930964Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.36589545Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.365931562Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.385328871Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.385364478Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.403512048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.403546859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.422318755Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.42234974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.442860612Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.442902501Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.461588343Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.461632444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.461856202Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.462005914Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551153184Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551204794Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551211206Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551218322Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551223835Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551229131Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551234541Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551240175Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551330126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551335573Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551340588Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551347086Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551352083Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551357052Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551363155Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551367824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551372661Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551378736Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551383687Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.551456335Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"84220936-2770-4154-832c-bef04d60dd6f","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized)]","latency_us":454033,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:30.551607773Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"160.407869ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"84220936-2770-4154-832c-bef04d60dd6f\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"84220936-2770-4154-832c-bef04d60dd6f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.713484458Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.713831979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.713844007Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.734917058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.734959378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.754537419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.754575493Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.77080147Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.77084213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.789228504Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.789268939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.806264913Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.80630862Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.823421563Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.823457264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.840456959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.840491366Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.855931373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.855960943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.875073514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.875105831Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.89426675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.894305245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.913932939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.913972961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.932629015Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.9326708Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.950236413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.950268822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.968804306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.968838993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.989004377Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:30.989042187Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.009589751Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.009628883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.032685551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.032729924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.050966695Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.051000793Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.069514794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.069557489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.089300061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.089342493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.08958443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.089815454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184784962Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184854901Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184862523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184869639Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184875292Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184881677Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184886823Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184892037Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.18489707Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184903859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184927737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184933419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184939841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184944603Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184949516Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184954269Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184959425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184964703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.184970807Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.185133875Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7517b818-03ba-404e-8806-d21f28f93bd7","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized)]","latency_us":470214,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.185293394Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"169.912786ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7517b818-03ba-404e-8806-d21f28f93bd7\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7517b818-03ba-404e-8806-d21f28f93bd7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.357208341Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.357555112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.357561797Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.38040775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.380463233Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.4018743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.401924458Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.421169545Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.421213843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.441339184Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.441379984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.459373306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.459407976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.477192497Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.477242063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.495472442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.495509304Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.513712014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.513749721Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.535431398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.535474252Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.555581151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.555623574Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.574875795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.574912396Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.594506217Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.59455613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.614286511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.614323637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.633886829Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.633927686Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.652943242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.652978718Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.673577399Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.67361264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.692190558Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.692226578Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.711130556Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.711167269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.729775437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.729813613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.749236279Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.749281703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.749500895Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.749710955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842710761Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.84282964Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842837609Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842843046Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842849646Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842854934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842861459Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842865247Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.84287031Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842876544Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842880388Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842887166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842893391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842916867Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842922426Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842927367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842933676Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842938289Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.842943248Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:31.843020406Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"df06e809-89bd-4436-ae89-006942c3fbda","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized)]","latency_us":484347,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:31.843174676Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"250.177432ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df06e809-89bd-4436-ae89-006942c3fbda\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df06e809-89bd-4436-ae89-006942c3fbda\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.095141032Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.095412172Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.095420075Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.118102321Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.118146947Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.137451453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.137499276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.15794115Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.157981755Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.177975602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.178013731Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.196909221Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.196944227Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.216927384Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.216966338Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.234549734Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.234600744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.251663831Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.251698809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.26883055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.268865041Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.287603568Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.287642072Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.306445445Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.306480273Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.325326327Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.325372406Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.343888665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.343924291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.36204928Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.362082853Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.380677993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.380711247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.398597604Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.398631683Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.418607973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.418643168Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.43606926Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.436101876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.454357429Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.45439598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.473556176Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.473600802Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.473809569Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.473948069Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564648254Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564713806Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564788548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564794417Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564801084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564806118Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564812671Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564816299Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564821409Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564826087Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564831187Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564836418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564842559Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564847819Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564852666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.5648589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564877398Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564883932Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.564888873Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.564971178Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2def43aa-5532-41bd-94d0-3d0e1c4163fa","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized)]","latency_us":468418,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:32.565123656Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"155.074076ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2def43aa-5532-41bd-94d0-3d0e1c4163fa\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2def43aa-5532-41bd-94d0-3d0e1c4163fa\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.721994238Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.722346694Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.722352938Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.743037828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.743079491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.762708605Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.762763927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.782418464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.782463944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.801115758Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.801150381Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.819591539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.819630371Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.837827649Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.837860847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.855622919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.855659088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.874071571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.874188236Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.897477789Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.897529747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.919492702Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.919532865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.939365397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.939400347Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.9607131Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.960755841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.981123443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:32.981163953Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.000493601Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.000533099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.02336493Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.023409856Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.0445165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.044561023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.066491781Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.066544086Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.086449149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.086483827Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.105724519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.105758457Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.124462323Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.124507052Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.124722288Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.124862176Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217575489Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217629726Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217637009Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217650332Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217655769Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217662444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217669073Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217674052Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217680295Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217685095Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217691442Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217696327Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217701398Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217706189Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217712142Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217717121Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217722034Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217728343Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217731981Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.217810817Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"35a38fd7-c471-4425-af67-fad9fe49b3a9","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized)]","latency_us":494320,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.217958026Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"207.727569ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35a38fd7-c471-4425-af67-fad9fe49b3a9\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35a38fd7-c471-4425-af67-fad9fe49b3a9\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.427995781Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.428318197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.428325669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.448736537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.448781323Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.466934155Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.46698195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.486408001Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.486447714Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.504541813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.504583185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.522504738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.522540945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.541010254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.541046058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.560199475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.560233404Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.578559858Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.57859404Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.599012164Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.599051174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.61902127Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.619059727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.63828833Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.63832974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.65809582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.658133471Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.678064929Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.678099393Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.697655057Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.697689923Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.71941067Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.71945133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.738021292Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.738055511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.757822465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.757863397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.776491911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.776540671Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.795702089Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.795735906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.814999195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.815044899Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.81530817Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.81543225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906669757Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906725211Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906732721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906738271Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906744524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906750047Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906765562Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906772284Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906777384Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906783779Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.90678871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906793876Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906798891Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906805276Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906810319Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906815457Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906820472Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906825507Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.906830589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:33.906932685Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d54d10e4-204d-418b-b524-805f8fadd978","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized)]","latency_us":477520,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:33.907093573Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"367.437872ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d54d10e4-204d-418b-b524-805f8fadd978\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d54d10e4-204d-418b-b524-805f8fadd978\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.276643635Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.276921703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.27693797Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.299131134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.299177776Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.317197711Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.317246107Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.335778494Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.335818461Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.353653676Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.35369173Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.371424851Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.371467123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.392578991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.392614861Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.413843168Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.413878203Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.43236955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.432403994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.451039033Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.45107269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.469964039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.470004746Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.488207719Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.48823996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.505944469Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.506053203Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.525599438Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.525640098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.544052838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.54416223Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.562871789Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.562918196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.58091372Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.580952778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.598983775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.599020432Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.617049472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.617083926Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.634997767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.635042918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.653124651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.653169239Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.653456328Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.653586034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747232427Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747279912Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.74728639Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747292493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747299449Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747305159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747310475Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747317066Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747322091Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747338347Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747344366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747350141Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747359348Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747367579Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747373232Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747378998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747386386Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747390821Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747396568Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:34.747477551Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fbce12b8-544b-4a2b-b228-747fcad8a807","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized)]","latency_us":469492,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:34.747721491Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"297.053905ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fbce12b8-544b-4a2b-b228-747fcad8a807\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fbce12b8-544b-4a2b-b228-747fcad8a807\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.047043797Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.047296178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.047304353Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.068986698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.069038242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.090681867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.090726905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.109388745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.109424268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.129632734Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.129740071Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.149904253Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.149943535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.16808246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.168120992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.18678197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.186888102Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.205047163Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.205084935Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.225085073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.225116827Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.243784626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.243816295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.261861237Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.261897332Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.27990017Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.279943802Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.298015192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.298048895Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.316133304Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.316234745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.334703772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.33475362Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.352725718Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.352760809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.37083263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.370871565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.388799881Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.388832019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.407070156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.407111201Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.425673624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.425715044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.42590043Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.425995241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524158444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.52420952Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524217733Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524224473Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.52422948Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524236033Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524241443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524247836Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.52425289Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524257999Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.52426309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524269719Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524285197Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524290653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524295847Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524300985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524307435Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524312685Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524316341Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.52443641Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"22852799-b765-47b6-ae9b-4601cae7abed","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized)]","latency_us":476013,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:35.524595685Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"262.637139ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"22852799-b765-47b6-ae9b-4601cae7abed\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"22852799-b765-47b6-ae9b-4601cae7abed\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.788825374Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.789112706Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.789125635Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.811151697Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.811188448Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.833046061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.833109278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.854893358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.854932996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.87458248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.874614947Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.893086598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.893120067Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.91236255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.912400098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.930371842Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.930405181Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.948107015Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.948139716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.968466902Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.968507525Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.987483675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:35.987519647Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.007426416Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.007473415Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.030860909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.030910916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.051596013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.051634339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.072172583Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.072211499Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.092788459Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.092824273Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.11286052Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.112903852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.132079601Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.132114149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.151622411Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.151660512Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.170781946Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.170815903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.189244757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.189286139Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.189491202Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.18969436Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282865859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282910101Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282917545Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282924473Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282929854Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282934683Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282939595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282944367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282950396Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282954979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282959659Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282964384Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282969305Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282974209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282977809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282989443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.282993137Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.28299775Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.283003938Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.283067098Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ad5e8e2b-369d-4585-99eb-cde017a68f24","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized)]","latency_us":492780,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:36.283221517Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"412.15885ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ad5e8e2b-369d-4585-99eb-cde017a68f24\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ad5e8e2b-369d-4585-99eb-cde017a68f24\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.698035596Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.69834313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.698355656Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.721092019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.721131245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.741895984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.741930485Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.760578904Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.760611877Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.778293173Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.778329343Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.795691802Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.795721952Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.81333454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.813366116Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.831831773Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.831865167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.851419159Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.851452431Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.871005793Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.871045882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.891982718Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.892014613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.911405905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.911439984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.9303164Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.930362244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.948273531Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.948312522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.966386582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.966419526Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.98442535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:36.98445627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.005224043Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.00525605Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.025772583Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.025872808Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.04281619Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.042914987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.062532457Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.062567024Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.082287586Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.082330477Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.08256183Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.082710126Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177528162Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177577559Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177601823Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177609135Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177614451Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.17762001Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177625148Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177630435Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177637129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177640439Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177643918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177648764Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177652186Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177656902Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177661721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177666661Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177669989Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177678718Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177682122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.177725379Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d72fc2c9-7686-42d5-9169-51c9aa9169bf","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized)]","latency_us":478299,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:37.177885141Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"666.609896ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d72fc2c9-7686-42d5-9169-51c9aa9169bf\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d72fc2c9-7686-42d5-9169-51c9aa9169bf\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.847357299Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.847618624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.84762482Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.871295522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.871341967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.894232062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.89428611Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.916116801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.916160339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.936367817Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.936403518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.956169436Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.956215581Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.975844013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.975885723Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.995081822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:37.995125538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.014572843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.014630098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.034859413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.034898039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.054454161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.054488971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.072910084Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.072944923Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.094285876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.094325149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.113625854Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.113682724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.133505144Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.133542148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.152741845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.152778905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.171388651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.17142423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.18926433Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.189297518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.208466479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.208518953Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.22773941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.227774905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.246989092Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.24703926Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.247282853Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.24738147Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342023777Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342077618Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342085118Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342088859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342093734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342117247Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342122619Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342127504Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342132322Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342137179Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.34214376Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342149123Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342152667Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342157373Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.34216102Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342165895Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342170798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342175439Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342178617Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.342232477Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9330f9bb-b40a-47af-85cc-6d25243ec9db","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized)]","latency_us":493472,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:38.342470979Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"653.731712ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9330f9bb-b40a-47af-85cc-6d25243ec9db\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9330f9bb-b40a-47af-85cc-6d25243ec9db\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.999232385Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.999575123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:38.999592523Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.020817001Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.020867627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.042000728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.042046686Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.061599722Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.061641507Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.079219537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.079259232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.097003837Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.097036134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.113915328Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.113945909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.131868028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.131899998Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.151050557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.15108008Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.1704458Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.170483544Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.188822442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.188854421Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.20711766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.207153136Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.226073815Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.226105241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.245454896Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.245492416Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.262854768Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.262950223Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.28069665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.28073071Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.296903572Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.29693421Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.313622521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.313654443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.332393594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.332425348Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.350542107Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.350577864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.368336576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.36837463Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.368570787Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.368689506Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459568248Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459628232Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459636735Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459642866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459650638Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459658232Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459665667Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459673054Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459699267Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459706993Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459712852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459718805Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459722865Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459728293Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459733815Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459739534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459745159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459749509Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459753494Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:39.459791679Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"29e1d1b7-e539-48d2-a1a9-1152f0984ad7","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized)]","latency_us":459175,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:39.459942339Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"798.541154ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"29e1d1b7-e539-48d2-a1a9-1152f0984ad7\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"29e1d1b7-e539-48d2-a1a9-1152f0984ad7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.261789878Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.262090288Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.262105373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.283005709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.283049735Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.302189156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.302223113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.320028231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.320064963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.339043245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.339078739Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.358465805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.358500295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.376183804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.37622063Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.394801256Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.394832973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.413282854Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.413319351Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.432452639Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.432488031Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.452246292Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.452284112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.470971934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.471010016Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.486951028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.486991164Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.505960882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.505998749Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.523920661Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.523955247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.543253525Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.543289742Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.562007598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.562038245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.581999036Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.582031174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.599200636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.599229614Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.619832587Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.619874148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.640106216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.640150411Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.64035601Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.640475318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730759568Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730799994Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730807185Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730812744Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730817638Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.73082271Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730827744Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730898059Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730903234Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730907979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730932166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730935616Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730940183Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730943595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730948358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730952923Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730957639Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730962195Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.730966536Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:40.731021971Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fc7f4fa0-1ecb-4d73-9b33-62f4a0054b35","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized)]","latency_us":467845,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:40.731186647Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.565844505s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fc7f4fa0-1ecb-4d73-9b33-62f4a0054b35\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fc7f4fa0-1ecb-4d73-9b33-62f4a0054b35\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.302348735Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.302605897Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.302612948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.325274329Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.32532053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.345338209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.345379403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.36532979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.365368678Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.385651145Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.385687915Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.40547747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.405512224Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.424582792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.424628862Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.443993819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.444027917Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.462668261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.462706859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.482929767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.482965683Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.501531338Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.501566514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.519890976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.519933324Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.53911182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.53915413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.557231082Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.557266314Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.575432505Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.575467775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.594316818Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.594353372Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.615713608Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.615748867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.635218401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.635254927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.654112946Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.654152547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.673521829Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.673558504Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.692982605Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.693043131Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.693292909Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.693387917Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788333633Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788389444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788394357Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788466846Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788470596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788475958Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788480871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788485989Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788492093Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788497259Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788503372Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788508519Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788513337Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788535735Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788541979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788546873Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788551654Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788554888Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.78855966Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:42.788634512Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1f6ca58e-fa0d-47f4-af0a-3de6c1736530","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized)]","latency_us":484880,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:42.788794594Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.641702666s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f6ca58e-fa0d-47f4-af0a-3de6c1736530\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1f6ca58e-fa0d-47f4-af0a-3de6c1736530\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.238075266Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"1.4065373s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.437046458Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.437332373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.437339058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.45816988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.458206499Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.477885278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.477914801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.497241724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.497277182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.517045553Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.51707906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.536816079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.536847316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.556786579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.55681928Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.576623994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.576701527Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.5965464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.596581605Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.616810482Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.616842029Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.637038071Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.637069393Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.658384819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.658422554Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.679239259Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.679271313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.698691457Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.698726023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.718845033Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.718878334Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.73929778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.73933245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.758207182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.758237408Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.777521161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.777556778Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.796526171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.796556058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.815657376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.815686786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.836710658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.836748974Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.837020214Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.837142026Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936411123Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936465827Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936473084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936479807Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936485122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936491619Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936496813Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936501894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936506797Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936512788Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.93651786Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936524226Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936529111Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936535129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936554536Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936559496Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936564437Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936569246Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936573952Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:44.936616478Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"90be6c4f-4f95-40b5-b72e-189d38da9665","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized)]","latency_us":498124,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:44.936766247Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"908.765523ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"90be6c4f-4f95-40b5-b72e-189d38da9665\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"90be6c4f-4f95-40b5-b72e-189d38da9665\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.848978842Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.849273026Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.849279149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.870824476Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.870865745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.890674333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.890715555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.909159856Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.909195153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.927788767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.927825453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.946181775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.946218151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.965747733Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.965783022Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.986288562Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:45.98634023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.005919142Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.005969994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.028049705Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.028105816Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.048153004Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.048189343Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.06754668Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.06758Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.087299424Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.087413532Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.106762867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.106800226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.125965133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.126008287Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.14584397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.145878883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.164959874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.164997441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.184676629Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.18471248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.204068154Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.204105814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.222839141Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.222873736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.241961365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.242007782Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.242199823Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.242333711Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338184604Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338249799Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338257177Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338262596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338268906Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338272881Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338278337Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338281862Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338287093Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.33829284Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338297725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.33830275Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.33830664Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338310269Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338315079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338318716Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338323441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338338132Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338342097Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:46.338348689Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c8a2767d-89f9-45d2-8eb3-40419353b1fe","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized)]","latency_us":487912,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:46.338498823Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.102977596s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c8a2767d-89f9-45d2-8eb3-40419353b1fe\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c8a2767d-89f9-45d2-8eb3-40419353b1fe\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.445301313Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.445562215Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.445569687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.46744592Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.467490875Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.487504582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.487547427Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.506397086Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.506435196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.526862491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.526900376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.54525981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.545362688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.563136965Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.56317546Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.580973385Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.581006742Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.600168012Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.600216154Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.620918957Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.620958098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.639912524Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.639951028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.658030278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.65806851Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.676401945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.676439971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.695005533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.695050168Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.713430822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.713465539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.731366936Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.731398465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.74974472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.749784902Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.768054139Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.768092539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.788067865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.788105112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.806542724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.806571413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.825840763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.825880411Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.826145446Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.826269508Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921521192Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.92158979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921597731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921604734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.92161846Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921623635Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921636404Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921641701Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921646754Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921656204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.92165987Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921665083Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921670154Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921675376Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921682051Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921686879Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921691277Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921694709Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921698318Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:47.921743488Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b4c0fc5b-77e3-4ed4-a6cc-767be1a81ee3","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized)]","latency_us":475059,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:47.921924673Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.691016024s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b4c0fc5b-77e3-4ed4-a6cc-767be1a81ee3\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b4c0fc5b-77e3-4ed4-a6cc-767be1a81ee3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.621772843Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.622026152Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.622035293Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.644454052Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.644502728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.664036424Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.664079925Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.682711254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.685031099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.700329142Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.702577153Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.720201539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.720248209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.73787845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.737990124Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.756414378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.756456613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.774405254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.774452046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.792240299Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.792301904Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.810244084Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.810287183Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.827986162Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.828032672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.845782526Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.84582251Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.863058363Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.863101358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.880927894Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.880970701Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.899125563Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.89917033Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.920322813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.920368368Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.938382029Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.938430976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.956017014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.958273117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.973885609Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.973929335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.991309208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.9913935Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.991676373Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:50.991808272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084197215Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.08424593Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084254555Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084267258Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084272517Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.08428537Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084290283Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084303531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084314162Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084319581Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084323106Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084328019Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084331628Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.08433655Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084341313Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084345954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.08434961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084352844Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084357794Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:51.084390289Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"0abe17fe-ae4a-4ec1-95ab-e2f2af523e7a","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized)]","latency_us":461300,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:51.084550455Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.19832715s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0abe17fe-ae4a-4ec1-95ab-e2f2af523e7a\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0abe17fe-ae4a-4ec1-95ab-e2f2af523e7a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.292978254Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.293268942Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.293281232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.314969098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.315013405Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.332751531Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.33280708Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.353047238Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.35309762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.372228966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.372275073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.39066401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.390718132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.408547489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.408605335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.427012934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.427071744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.445251616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.445291629Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.464196044Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.464264763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.482809313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.482857454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.501249232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.501300467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.519556717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.519613616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.538599962Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.538643219Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.557437246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.557491875Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.575979235Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.578252057Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.594546633Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.59459319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.612856099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.612898512Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.630638141Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.63068198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.649704975Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.64976822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.667625287Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.667691035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.667904857Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.668024971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762706637Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762765081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762772367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762778713Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.76278416Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.76278911Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762795392Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762800417Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762805217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762810074Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762887141Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762892748Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762898101Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.76290321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762908404Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762913664Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762917321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762920666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.762924755Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:54.762971256Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c4cd1dcd-5d5b-4cfb-b41c-2f775bae57a0","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\", TimeNotSynchronized), (\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\", TimeNotSynchronized), (\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\", TimeNotSynchronized), (\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\", TimeNotSynchronized), (\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\", TimeNotSynchronized), (\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\", TimeNotSynchronized), (\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\", TimeNotSynchronized), (\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\", TimeNotSynchronized), (\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\", TimeNotSynchronized), (\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\", TimeNotSynchronized), (\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\", TimeNotSynchronized), (\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\", TimeNotSynchronized), (\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\", TimeNotSynchronized), (\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\", TimeNotSynchronized), (\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\", TimeNotSynchronized), (\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\", TimeNotSynchronized), (\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\", TimeNotSynchronized), (\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\", TimeNotSynchronized)]","latency_us":468607,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:55:54.763146028Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.826498044s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4cd1dcd-5d5b-4cfb-b41c-2f775bae57a0\", \"content-length\": \"124\", \"date\": \"Fri, 18 Oct 2024 17:55:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c4cd1dcd-5d5b-4cfb-b41c-2f775bae57a0\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 64822dce-d64a-403f-bf53-d84635424320 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 640c31ba-0a33-4d06-bcaa-280a64058d07, kind: Service { id: 64822dce-d64a-403f-bf53-d84635424320 }, name: Name(\"ntp-64822dce-d64a-403f-bf53-d84635424320\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 176, 131])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 03d0126b-4566-466f-bb9c-de9896115c34 (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 69bfa368-1d69-4d23-88e9-3383e573f0eb (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68422c74-fc22-434f-add8-652fc427fc03 (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d1c58db-a735-48db-90ac-891756bdde5d (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1213c740-3b34-4c05-9e32-d5fae22eaaf8 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c8bd7410-7132-42bd-a55f-2a198a6b2acd (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: c4709288-9135-419c-a897-87bc98b8ce39 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 559ebce8-c652-447b-8465-50ce9559421e (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 2a34aa70-afc1-4c49-b577-50951425e4ca (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 8f12f963-ce2b-4e5c-ac07-73f657227673 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External } } } }, OmicronZoneConfig { id: b698f337-7ba4-4ae4-8547-b207e6fcae83 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External } } } }, OmicronZoneConfig { id: 8efe69b7-0efd-43f2-8275-553df5dac16e (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4736466c-52e2-4b18-8ead-0adae93f7d49 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3925fdff-229a-4176-baab-54ab3ed1bb83 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6322d41-765c-4dcb-a58b-608a1681fd63 (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External } } } }, OmicronZoneConfig { id: e87bc7fc-d008-4cd1-83dd-c64d8728793a (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External } } } }, OmicronZoneConfig { id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615 (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External } } } }, OmicronZoneConfig { id: a46900f5-d79b-4c04-ad72-7cd4485153fd (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.600740691Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.601080346Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.601087293Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5a24845a-5b82-4ab6-b0ce-11d63d68b9cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.623477948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.625746669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1fb67d4e-bac4-4e8f-b10a-4ef050bccb88, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.643122498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.643164686Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 7a37cb7b-e967-4955-ad9c-4441ce0d32e6, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.661768108Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.661800508Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 03d0126b-4566-466f-bb9c-de9896115c34, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.674503862Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65101"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.674797241Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"9746584c-8ee8-4064-8077-2be9a01e2b5e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65101","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.681713275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.681723813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8e79611e-ba88-4a64-8bf6-28d7df64e66a, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.7002289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.700284813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 69bfa368-1d69-4d23-88e9-3383e573f0eb, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.719661475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.719705847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68422c74-fc22-434f-add8-652fc427fc03, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.738077944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.738127707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2d1c58db-a735-48db-90ac-891756bdde5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Cockroach }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.755594367Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.75570672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 68483b33-33b2-4e6b-bcd7-72f28a004330, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.77441738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.774471335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.792362232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.794592824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c8bd7410-7132-42bd-a55f-2a198a6b2acd, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.811159247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.811210464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8f12f963-ce2b-4e5c-ac07-73f657227673, dataset_name: DatasetName { pool_name: ZpoolName { id: 8853a1c1-d102-4f80-9012-356473e594ff (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.829316984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.829369738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b698f337-7ba4-4ae4-8547-b207e6fcae83, dataset_name: DatasetName { pool_name: ZpoolName { id: 8523f2c5-b045-4f20-8f22-89f361c6e33c (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.847641285Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.84768278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 8efe69b7-0efd-43f2-8275-553df5dac16e, dataset_name: DatasetName { pool_name: ZpoolName { id: 24197587-83af-46c0-a36e-5e4a26350d21 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.865302587Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.865347241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4736466c-52e2-4b18-8ead-0adae93f7d49, dataset_name: DatasetName { pool_name: ZpoolName { id: 956d8649-5e02-4c83-a766-709d6c1c19f4 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.884108727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.884154338Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3925fdff-229a-4176-baab-54ab3ed1bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e3d3772-4d1a-4cd4-9451-d3f5402e0376 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.902772941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.902819179Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6322d41-765c-4dcb-a58b-608a1681fd63, dataset_name: DatasetName { pool_name: ZpoolName { id: aa3e117f-01a2-4173-8076-71ed9651de49 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.921468397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.921511738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e87bc7fc-d008-4cd1-83dd-c64d8728793a, dataset_name: DatasetName { pool_name: ZpoolName { id: e09cb849-5302-41dd-87a5-36e8723c48fb (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.940372376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.942375547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 06f6275f-6e08-4df0-8541-1b7a4e8b6615, dataset_name: DatasetName { pool_name: ZpoolName { id: c6223f26-38f9-4483-9d17-efd8e60dd490 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.958765998Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.958821958Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a46900f5-d79b-4c04-ad72-7cd4485153fd, dataset_name: DatasetName { pool_name: ZpoolName { id: 883d1cbf-e1f8-4c86-b18f-d626feb923fd (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.977969626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.97802961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.97820636Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:58.978394438Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.068937153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.15669589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.156925095Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.156945402Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.156956746Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.157000425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.157042436Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.157051154Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.157067851Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.157074358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.35318631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.353560315Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.353578616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.353591685Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.353602466Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.353637248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.354673233Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.366361315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.377658415Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.378006029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.382535004Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.386906458Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.391180219Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.395801955Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.400550047Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.405341248Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.410245977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.525999026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.531147176Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.535911862Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.540439567Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.544879553Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.549363092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.554019703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.558755306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.563655812Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.56872228Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.573957161Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.579841564Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.58555013Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.592271271Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.597385322Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.604078413Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.609748768Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.614050928Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.617676455Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.623970642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.663066169Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.698111981Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.733493615Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.77032488Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.812337405Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.813671154Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.813683793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.855423191Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 187, 180]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.855447716Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.856811691Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.856822877Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.914062079Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 222, 198]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.914118686Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.91576626Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.915875545Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.976668066Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 173, 175]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.976691289Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.9791911Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:55:59.979206662Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.030150462Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 228, 68]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.030181194Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.031707323Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.031724396Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.082708007Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 185, 152]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.082738213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"}
{"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.082962253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.082972172Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.083044774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.083053811Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.0830984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.083106265Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.08386593Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.086025261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.133108808Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.136619829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.13707073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.139925676Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.139937788Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.140058119Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"}
{"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.140069463Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.140221538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.293917519Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.297632842Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.300681555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.300700437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.301062268Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.315673644Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.326618874Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.462127694Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.475424487Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.485919656Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.502958578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.517927155Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.544122059Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:00.575126287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:00.738629759Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"1.830849616s"}
{"msg":"Profile for oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService10'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::19'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_c6223f26-38f9-4483-9d17-efd8e60dd490/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::19'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='1ff9245b-2972-419d-ac27-73880f962c55'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.434144027Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.43422096Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService13'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::14'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_24197587-83af-46c0-a36e-5e4a26350d21/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::14'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='0f029078-a2f2-48a8-a6b0-b9b3bf0b5295'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.527168469Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.527380377Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService15'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::f'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::f'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.554470572Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.554635388Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService19'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::11'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::11'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.572844737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.573119637Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService16'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::1a'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_883d1cbf-e1f8-4c86-b18f-d626feb923fd/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::1a'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='821c0d59-ad53-42c2-a45f-00721d2b7a8a'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.650454255Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.650682701Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService12'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::17'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_aa3e117f-01a2-4173-8076-71ed9651de49/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::17'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='25938cff-45d3-47cb-968f-bd46a3ceff43'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.658703695Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.658866008Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService17'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::10'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::10'/>\n        <propval type=\"astring\" name=\"listen_port\" value='17000'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.665464106Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.66568162Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService21'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::13'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_8523f2c5-b045-4f20-8f22-89f361c6e33c/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::13'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='392f6388-eaeb-4772-b2b4-4748e57549c6'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.673723879Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.673942443Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService11'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::15'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_956d8649-5e02-4c83-a766-709d6c1c19f4/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::15'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='3f74f18a-4eb6-49f9-b5fa-a193ab562388'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.784021351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.784261854Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService20'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::12'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_8853a1c1-d102-4f80-9012-356473e594ff/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::12'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='5ff38177-a0f1-4560-9abb-ebc164c042c0'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.80523645Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.805846817Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService23'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::18'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_e09cb849-5302-41dd-87a5-36e8723c48fb/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::18'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='afdd6719-743c-4e79-b6b3-f40a2d8d8eb1'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.869297925Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:02.869704097Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 54.549821ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:02.990678875Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 58.662436ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.001540505Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 45.092125ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.010524867Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService18'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::d'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value='1213c740-3b34-4c05-9e32-d5fae22eaaf8'/>\n        <propval type=\"astring\" name=\"address\" value='[fd00:1122:3344:101::d]:12223'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:03.010747125Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 71.020979ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.019246326Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:03.019599847Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 54.706906ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.033609226Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 40.35853ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.041386867Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService22'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::16'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value='oxp_4e3d3772-4d1a-4cd4-9451-d3f5402e0376/crucible'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::16'/>\n        <propval type=\"astring\" name=\"listen_port\" value='32345'/>\n        <propval type=\"astring\" name=\"uuid\" value='285b6ca2-a27b-4d54-bd1f-900c07b48d3d'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:03.041622099Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 44.01469ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.051074985Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 32.664976ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.060201741Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 53.790346ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.083824895Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 78.875383ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.20827084Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 51.728454ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.309476736Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 98.214274ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.342904976Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 69.922334ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.386522373Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 105.320773ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.431953683Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 107.180539ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.495117792Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 146.384581ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.660542374Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Booting oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:03.662061209Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 30.21898ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.763482447Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 54.514395ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.798676632Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 104.914147ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.827874805Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 146.58859ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.867019616Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 203.313621ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:03.904249001Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 52.004071ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.16013624Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 196.090389ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.207694105Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 235.995066ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.370158539Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 26.965202ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.413764598Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 224.404766ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.442827325Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 175.391253ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.484707038Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 183.030928ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.501077475Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 110.334343ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.539356232Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 113.703765ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.558249245Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 290.448607ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.571166771Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 230.006656ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.584106412Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 332.606301ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.606930515Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 276.168599ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.647879795Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 501.344648ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.714823583Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 426.246184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.815658459Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 85.967449ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.827554299Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 474.436154ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.838744975Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 491.529807ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.850426477Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 479.226867ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.861440675Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 32.312381ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.870398036Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 134.663401ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.879819858Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 414.499303ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.892044656Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 295.819656ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.904096177Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 490.496701ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.915144773Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 166.795617ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.926806983Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 53.619532ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.940648694Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 406.626864ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.950528584Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 956.104346ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:04.962168938Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 257.952568ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.007855299Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 218.327871ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.027005359Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 440.457826ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.107117869Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 712.921009ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.215087558Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 775.515933ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.233518986Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 555.767812ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.260860849Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 547.662622ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.271799386Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 277.54011ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.285145189Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.063263529s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.321071016Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 592.6019ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.333077006Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 855.273832ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.360481517Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 997.311419ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.37634736Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.127656576s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.391493966Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 981.573531ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.419448642Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 650.05281ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.562840103Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.182337861s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.579695056Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 1.324972928s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.838293239Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 1.44627389s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.859692408Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 828.477998ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.94151244Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 882.192497ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:05.961068634Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 989.31931ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:10.716772919Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.014791919s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.122018066Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 908.249294ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.208760587Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 656.457625ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.222044468Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 1.153607342s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.23453049Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 1.479910348s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.246244095Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 1.464553758s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.256248415Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 1.408655719s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.267150869Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 877.182248ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.277679636Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 968.534955ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.287320964Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 855.480949ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.297241934Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 963.561088ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.309166104Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 619.486706ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.319325253Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 981.514787ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.722170957Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 1.441745767s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.893780573Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 1.291119658s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:11.955424726Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 684.271739ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.131660813Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.222894581s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.152092591Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 1.322810802s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.166761309Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 799.802053ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.178223978Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 609.597533ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.272618255Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.29278357s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.286985682Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 1.394419909s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.404645588Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 875.796788ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.695524526Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 863.467065ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.716993366Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 996.5849ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.739310092Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 1.332402801s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.752152247Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 963.765378ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.832213874Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 592.331117ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.898413067Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 754.718098ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:12.994689914Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 683.623294ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.264795046Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 676.504687ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.354132271Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 940.352157ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.391776421Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.2198965s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.506495418Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 905.183568ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.519728969Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 831.584551ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.586175895Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 936.055378ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.59817038Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.255323094s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.60909531Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 1.242380455s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.752855308Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 849.540906ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.76508496Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 575.747761ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.810373991Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 634.216343ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.820698367Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 602.254136ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:13.967021391Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 1.390796794s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.049258162Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 806.628789ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.104609612Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.163578556s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.351920539Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 854.94387ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.40751809Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 1.11396737s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.445904276Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 906.851837ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.512277223Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 912.04159ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.532144388Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 520.152239ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.562667436Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 1.282708251s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.591795273Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 944.932233ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.634845195Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 852.096992ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.748307256Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 842.036261ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.894894962Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 932.955218ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:14.9340594Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 1.439315313s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.027559152Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 1.417269507s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.104580316Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 637.537424ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.285204001Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 1.053654599s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.43968556Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 1.083861717s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.458709143Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 1.251367927s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.476656342Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService14'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::e'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::e'/>\n        <propval type=\"astring\" name=\"listen_port\" value='8123'/>\n        <propval type=\"astring\" name=\"store\" value='/data'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse-admin\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::e]:8888'/>\n        <propval type=\"astring\" name=\"ch_address\" value='[fd00:1122:3344:101::e]:8123'/>\n        <propval type=\"astring\" name=\"ch_binary\" value='/opt/oxide//opt/oxide/clickhouse/clickhouse'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:15.485086184Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:15.486347523Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 628.084092ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.535206536Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 996.154215ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.581322456Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 555.677892ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.600869071Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.426551856s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.620497858Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.35682782s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.753628161Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 71.188605ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.80582103Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 537.370083ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.895935902Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 85.681233ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.916939351Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 1.168889014s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.935298139Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 1.398142192s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:15.95596062Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 133.611195ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.02223907Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 376.098368ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.177536397Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 1.469884694s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.197685196Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.273202097s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.236754616Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 765.321543ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.448250693Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 737.700536ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.480213204Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 681.086349ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.505439856Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 1.236119881s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.532748908Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 908.272589ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.552451237Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 1.01168372s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.565524705Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 723.780113ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.587904135Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 1.127847759s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:16.739929858Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService25'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::9'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte2'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.1.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.1.6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::9]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='172.30.1.6:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:16.822184331Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService28'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::8'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte5'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.1.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.1.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value='[fd00:1122:3344:101::8]:5353'/>\n        <propval type=\"astring\" name=\"dns_address\" value='172.30.1.5:53'/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:16.823243691Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:16.823278979Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:16.830330637Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.206120532s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.069946885Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 690.095848ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.118988082Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.173936596s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.131035176Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 29.33532ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.161856159Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 48.361391ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.171953969Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 680.646397ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.197969841Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 72.94052ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.205565264Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 1.389658624s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.221899696Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 838.468823ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.233740436Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 73.190131ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.246791083Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 644.697235ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.40909307Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 235.171294ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.470862697Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 278.258609ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.497024273Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 1.401549355s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.510195405Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 701.897115ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.520877827Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.477992526s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.533872313Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 1.01199569s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.589718055Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 504.038208ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.682720214Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.76816855Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"3.552245766s"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 428.539499ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.783922259Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 1.323132198s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.796920435Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 423.295429ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.812254344Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 931.945944ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.824404821Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 783.691905ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.881991503Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 1.341888637s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:17.892541103Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 826.067804ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.065452442Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 687.715026ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.082710425Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 615.390594ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.202758289Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 1.091236781s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.228149428Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 789.718585ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.240135561Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 775.545743ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.252889151Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.498642385s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.289352692Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.09927849s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.318557927Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService26'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::b'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte3'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.6'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:18.407653887Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService24'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::a'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte1'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.5'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:18.407702648Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:18.408170168Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Profile for oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlService27'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"static_addr\" value='fd00:1122:3344:101::c'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value='opte4'/>\n        <propval type=\"astring\" name=\"gateway\" value='172.30.2.1'/>\n        <propval type=\"astring\" name=\"ip\" value='172.30.2.7'/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:18.416926808Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:18.417302896Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"Booting oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:18.424845078Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 778.458645ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.621795025Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 1.212309247s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.637623535Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 59.141342ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.647679014Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 69.912396ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.660445555Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 1.07155043s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.682421786Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 67.135992ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.692595697Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 75.981095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.720783039Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 97.973087ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.886361991Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 142.135766ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.904794981Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 1.133976733s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.929816191Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 632.474689ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:18.968449681Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 677.00549ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.00396883Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 266.579829ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.091611376Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 63.521747ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.225731322Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 1.302812395s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.247907337Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 1.396134562s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.265529928Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 569.444447ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.28489615Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 595.413355ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.300483794Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 775.377458ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.316888218Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 777.668503ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.332950824Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 738.367095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.351761983Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 1.270860846s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.399756906Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 1.201212874s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.460828477Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 765.482368ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.480032804Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 354.396092ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.494035043Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 100.236025ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.512249238Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 1.191054381s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.524281002Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 1.258286131s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.614144619Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 229.650182ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.626076131Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 1.391702687s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.696039355Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 935.514562ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.765428876Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.488197048s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.803068593Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 654.53862ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.860844679Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 652.910545ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.873826695Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 1.125402398s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.886650107Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 728.474459ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.897581026Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.287574542s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:19.912367903Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 503.392407ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.076808658Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 1.187164577s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.101618014Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 982.639006ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.115132442Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 745.036146ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.12646838Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83\") failed: Property not found. retry in 1.260470163s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.260324879Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 1.134175142s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.529523511Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 873.314593ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.54002899Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 1.328536604s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.560819384Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd\") failed: Property not found. retry in 1.439697268s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.592771031Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 1.449637828s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.639917456Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a\") failed: Property not found. retry in 877.436397ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.678193737Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca\") failed: Property not found. retry in 1.029750605s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.689673219Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83\") failed: Property not found. retry in 1.221966967s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.702960932Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e\") failed: Property not found. retry in 940.312274ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.716902749Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 1.379308498s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.7283462Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63\") failed: Property not found. retry in 946.889249ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.883808501Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e\") failed: Property not found. retry in 533.545462ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:20.898372123Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39\") failed: Property not found. retry in 739.35047ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.024202829Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8\") failed: Property not found. retry in 1.054632198s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.10151414Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49\") failed: Property not found. retry in 741.464044ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.115902158Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615\") failed: Property not found. retry in 1.209058182s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.223929349Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 1.108388966s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.307694287Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673\") failed: Property not found. retry in 1.24029731s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.324204383Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 1.362454223s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.456173689Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 1.494619558s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:21.829466342Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 768.020972ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:22.096884794Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 645.154917ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:22.144892888Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 668.694254ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:22.30390881Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 1.366620589s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:22.441088465Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 818.725372ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:22.860524189Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 536.502235ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:22.917673421Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 1.051966836s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:23.084719887Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 1.424437543s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:23.170746836Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 1.389105989s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:23.35041987Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 921.047716ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:23.479998161Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 563.232735ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:23.704047146Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 571.390468ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:23.827973609Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 959.253747ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:24.149889383Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 1.224722709s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:24.283092667Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 630.956067ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:24.416241151Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 892.907522ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:24.429353089Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 1.01970516s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:24.609070543Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 1.237427967s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:24.755149971Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 1.14202864s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:25.059906564Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 877.547397ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:25.121064057Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b\") failed: Property not found. retry in 1.367653821s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:25.334554431Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd\") failed: Property not found. retry in 1.212097347s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:25.518729737Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 649.573113ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:25.641687408Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f\") failed: Property not found. retry in 1.190194426s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:26.00567062Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330\") failed: Property not found. retry in 634.214518ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:26.01753407Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\") failed: Property not found. retry in 941.807834ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:26.213926564Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2\") failed: Property not found. retry in 849.912338ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:26.303052785Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:27.072232608Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"4.203115989s"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237243756Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_30ebfc02-3884-4b1b-b0ab-82f344558c6b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237302208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_06f6275f-6e08-4df0-8541-1b7a4e8b6615"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237310098Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_4736466c-52e2-4b18-8ead-0adae93f7d49"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237316398Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_d6322d41-765c-4dcb-a58b-608a1681fd63"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237323658Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_8efe69b7-0efd-43f2-8275-553df5dac16e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237329918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_clickhouse_c8bd7410-7132-42bd-a55f-2a198a6b2acd"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237335889Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_c4709288-9135-419c-a897-87bc98b8ce39"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237342349Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_a46900f5-d79b-4c04-ad72-7cd4485153fd"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237348689Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_559ebce8-c652-447b-8465-50ce9559421e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237362229Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_external_dns_28a703d2-5c3e-4a89-8c59-9a6b18fe5d09"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.23736865Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_oximeter_1213c740-3b34-4c05-9e32-d5fae22eaaf8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.23737495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_2a34aa70-afc1-4c49-b577-50951425e4ca"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.23738041Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_8f12f963-ce2b-4e5c-ac07-73f657227673"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.23739105Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_b698f337-7ba4-4ae4-8547-b207e6fcae83"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.23739685Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_3925fdff-229a-4176-baab-54ab3ed1bb83"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.23740223Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_c5f97bb5-dce0-4358-bd22-da6cb1e4812f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.2374081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_c3438018-9a74-4aff-bf04-5f7fce86d0b2"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237518253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_external_dns_68483b33-33b2-4e6b-bcd7-72f28a004330"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.237534543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_e87bc7fc-d008-4cd1-83dd-c64d8728793a"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.238608538Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.240624484Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.246010356Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"57724a7d-df2a-42ab-95db-7bea6f34fb30","remote_addr":"[fd00:1122:3344:101::1]:34297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":28601152,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.255983943Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1321"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.256802502Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.257018167Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.264500597Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:707"}
{"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.266727358Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:719"}
{"msg":"rack_network_config: RackNetworkConfigV2 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.1.174,\n    infra_ip_last: 10.151.1.174,\n    ports: [\n        PortConfigV2 {\n            addresses: [\n                UplinkAddressConfig {\n                    address: V4(\n                        Ipv4Net {\n                            addr: 10.151.1.174,\n                            width: 24,\n                        },\n                    ),\n                    vlan_id: None,\n                },\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            lldp: None,\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Net {\n                            addr: 0.0.0.0,\n                            width: 0,\n                        },\n                    ),\n                    nexthop: 10.151.1.100,\n                    rib_priority: None,\n                    vlan_id: None,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Net {\n        addr: fd00:1122:3344:100::,\n        width: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.318498678Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:886"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.319578202Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.389498715Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.563916728Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.728519068Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:27.962271663Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:28.208670156Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:28.580302362Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:28.789827645Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:28.824653089Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59095"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:28.828324242Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"fe0efb77-a0b9-4cdc-b3de-ab757d20b4bc","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59095","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1273,"response_code":"200"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:29.0442299Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:29.501879436Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:30.194477404Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:31.004105558Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T17:56:31.312374501Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"9.002000096s"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:31.461771814Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:31.908608434Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:32.56990751Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:33.407107974Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/33bc13e3-6beb-4d1b-8b04-6e3da281e5d7/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:35.503963705Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fdb30928-805b-47ba-8ff3-3da6f3f58ad7\", \"content-length\": \"173\", \"date\": \"Fri, 18 Oct 2024 17:56:38 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"33bc13e3-6beb-4d1b-8b04-6e3da281e5d7\\\"\", request_id: \"fdb30928-805b-47ba-8ff3-3da6f3f58ad7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:38.318181474Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4c4862b-d825-4e26-91a5-efd02fb9ff41\", \"content-length\": \"173\", \"date\": \"Fri, 18 Oct 2024 17:56:41 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"08066c3d-b7ed-49d2-8578-f8769fb94736\\\"\", request_id: \"c4c4862b-d825-4e26-91a5-efd02fb9ff41\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:41.623494609Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a72bda82-1ffb-4404-b39a-eb15e4ecdeef\", \"content-length\": \"173\", \"date\": \"Fri, 18 Oct 2024 17:56:43 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"08066c3d-b7ed-49d2-8578-f8769fb94736\\\"\", request_id: \"a72bda82-1ffb-4404-b39a-eb15e4ecdeef\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:43.625139236Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a0accce-cdac-45d5-b471-4e4442331f6b\", \"content-length\": \"173\", \"date\": \"Fri, 18 Oct 2024 17:56:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"08066c3d-b7ed-49d2-8578-f8769fb94736\\\"\", request_id: \"6a0accce-cdac-45d5-b471-4e4442331f6b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:48.671174322Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:948"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:48.97267955Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:49.05788254Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33841"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:49.059046197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:49.059060257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:49.059084388Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:49.060336176Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:49.251045841Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"45205433-5d97-4555-8d06-a098588ad2c5","remote_addr":"[fd00:1122:3344:101::a]:33841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192741,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.299304926Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52914"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.301411804Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.302322874Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.303112412Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.303126653Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.303134133Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.303142273Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.303155383Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.303184384Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"57ba22df-059e-4ef0-b533-c1de7d534e29","remote_addr":"[fd00:1122:3344:101::a]:52914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3779,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.304355711Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.489971058Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59030"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.490082581Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.490092621Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Reading ledger from /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.490126452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.490417169Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:54.670587573Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"a0f0d445-343f-4b2e-b971-d4b8a7068e28","remote_addr":"[fd00:1122:3344:101::a]:59030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180519,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139101015Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55381"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139350401Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139400622Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139436313Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139443163Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139455923Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139469924Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139483334Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.139507294Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ddc2ea25-fbe4-4d99-b850-1eebb528702d","remote_addr":"[fd00:1122:3344:101::a]:55381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":330,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427033534Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56298"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427248119Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.42730181Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427333651Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427344611Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427356821Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427374102Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427389352Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.427412432Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5f205570-d0c1-4499-8dde-ce83296d29ab","remote_addr":"[fd00:1122:3344:101::b]:56298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":282,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.533687493Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61960"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.533917489Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 36ba59af-bf85-4423-a357-51f0d41b09ff, kind: Service { id: 68483b33-33b2-4e6b-bcd7-72f28a004330 }, name: Name(\"external-dns-68483b33-33b2-4e6b-bcd7-72f28a004330\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 152])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f1929d9e-6f3d-4be3-b8e4-711b38867554, kind: Service { id: 28a703d2-5c3e-4a89-8c59-9a6b18fe5d09 }, name: Name(\"external-dns-28a703d2-5c3e-4a89-8c59-9a6b18fe5d09\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 198])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 53cfda59-d0cc-456d-8218-38fa278d1ca8, kind: Service { id: 30ebfc02-3884-4b1b-b0ab-82f344558c6b }, name: Name(\"nexus-30ebfc02-3884-4b1b-b0ab-82f344558c6b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 180])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 8be92f42-3bcf-4463-9be3-fc3eaa3e55bb, kind: Service { id: c3438018-9a74-4aff-bf04-5f7fce86d0b2 }, name: Name(\"nexus-c3438018-9a74-4aff-bf04-5f7fce86d0b2\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 68])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f67c0489-2953-40c9-ae7c-0422d543c65f, kind: Service { id: c5f97bb5-dce0-4358-bd22-da6cb1e4812f }, name: Name(\"nexus-c5f97bb5-dce0-4358-bd22-da6cb1e4812f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 173, 175])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.53396662Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.53399172Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.534001651Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.534018161Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.534036641Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.534057762Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:55.534075522Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1ff159d2-eff4-45ed-9d13-5b7150c7fa41","remote_addr":"[fd00:1122:3344:101::c]:61960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":298,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:58.869521599Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52047"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:56:58.869645632Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"cfeb2b28-858f-42b2-8177-c5db15b52da3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52047","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:00.940573775Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61022"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:00.940732919Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"61bc5ee6-47f1-4e5c-ba7d-74c72a51e3cd","remote_addr":"[fd00:1122:3344:101::a]:61022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:00.982254835Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:00.982284056Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"362a0b46-b58b-4994-b16d-59d51de818f4","remote_addr":"[fd00:1122:3344:101::a]:61022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:01.077627067Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"660d326b-e632-45ca-adcc-d17c79811109","remote_addr":"[fd00:1122:3344:101::a]:61022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2155,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.76522358Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42452"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.766184062Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"043506ec-df89-4b90-af21-2c8fb704cfc8","remote_addr":"[fd00:1122:3344:101::a]:42452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":324,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.832811249Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41607"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.832901411Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"10f69ba2-41e8-470a-a1f8-1156fa199ef1","remote_addr":"[fd00:1122:3344:101::a]:41607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.900971182Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.901001933Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2d1457d5-a634-49d5-9c38-a8c03a41df80","remote_addr":"[fd00:1122:3344:101::a]:41607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.911575824Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3922e2ae-1cb4-40e0-8576-110ee15c25b5","remote_addr":"[fd00:1122:3344:101::a]:41607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":678,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.973465143Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37640"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:08.973596716Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.035756652Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.119224634Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"96f81eb7-5e15-4202-a275-196902e4f8e8","remote_addr":"[fd00:1122:3344:101::a]:37640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145686,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.120034522Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"06751427-6ed0-4a4c-998e-2f41937ab8c7","remote_addr":"[fd00:1122:3344:101::a]:37640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":521,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.216041059Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.216159402Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.218084476Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.22310166Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.22312447Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"4ed0a662-935e-4ff3-9d8a-02753715b3ee","remote_addr":"[fd00:1122:3344:101::a]:37640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8554,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.264049883Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55945"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.264208696Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e663f0f5-646a-4d9c-af45-6c530c8b88dd","remote_addr":"[fd00:1122:3344:101::b]:55945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.313031708Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.313059219Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ab996ead-163e-4a81-a154-070f3a081c2c","remote_addr":"[fd00:1122:3344:101::b]:55945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.32848006Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c13b9f44-e1e3-410d-9c9d-cccb0ca26c5a","remote_addr":"[fd00:1122:3344:101::b]:55945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":229,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.332474711Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.332617425Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"4dc1fd33-5aac-414f-8f59-d23450458ba7","remote_addr":"[fd00:1122:3344:101::b]:36200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.413241191Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58391"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.413348793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.44921301Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.515205783Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"87d82781-d7d0-4657-906e-2a4ad28818e9","remote_addr":"[fd00:1122:3344:101::b]:58391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101905,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.515559792Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e6f077f3-aa44-44b7-8482-40542ca5076e","remote_addr":"[fd00:1122:3344:101::b]:58391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.606178166Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52669"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.606499913Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"960e9dae-92a5-4ec6-99f1-8f24592bcdf2","remote_addr":"[fd00:1122:3344:101::c]:52669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.654933656Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.654973077Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"69021c09-885b-46f7-b4ec-4155d11f7e47","remote_addr":"[fd00:1122:3344:101::c]:52669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.670573783Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1dc02d86-fa46-46c6-99df-562651b416da","remote_addr":"[fd00:1122:3344:101::c]:52669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":405,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.792477579Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37565"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.792709295Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1d3d02a8-5aa3-43ec-acfc-562ffc3e72a9","remote_addr":"[fd00:1122:3344:101::c]:37565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.995900853Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42035"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:09.996010225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:10.032318663Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:10.08839117Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"517782c2-c66b-4b0c-82e4-9277601450c8","remote_addr":"[fd00:1122:3344:101::c]:42035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92428,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:10.088747088Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a016d6d0-041e-45ac-9221-ee35e9bf4512","remote_addr":"[fd00:1122:3344:101::c]:42035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:10.442632759Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:37310"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:10.684475578Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"8e6f3da3-3b5d-4ef6-8676-3af68fc909d6","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":241347,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:28.904698722Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57868"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:28.904828165Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"80acaca2-da35-4d41-b7fc-2a3dbb4c7b1e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57868","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.770407332Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61100"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.770571886Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3a801783-ed08-49d6-bdbb-63c1f1bdb0fe","remote_addr":"[fd00:1122:3344:101::a]:61100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":82,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.81641324Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.816477332Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0a13a834-f3aa-4b54-810d-007aaa1f8d26","remote_addr":"[fd00:1122:3344:101::a]:61100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.829206891Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7112545c-29a8-4170-b39f-71c53654e8b2","remote_addr":"[fd00:1122:3344:101::a]:61100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":272,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.977932379Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53322"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:38.978102243Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"530f7ea4-74ef-4f60-9a39-567ae78efd1e","remote_addr":"[fd00:1122:3344:101::a]:53322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.132634483Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48469"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.132750296Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.180835261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.249072976Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c7a9f7a3-1889-45c0-8628-db9dd7983939","remote_addr":"[fd00:1122:3344:101::a]:48469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116370,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.249407633Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0d9a2e75-02aa-445b-b979-2f881ea9eaf8","remote_addr":"[fd00:1122:3344:101::a]:48469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.369676393Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44718"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.369832986Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7a022a7c-9813-4b06-811d-41cc99ec1941","remote_addr":"[fd00:1122:3344:101::b]:44718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.385264538Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.385293629Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0d0058e7-0c15-4ba5-aafc-00763e6363f3","remote_addr":"[fd00:1122:3344:101::b]:44718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.396171386Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9cf85a39-f452-456f-9bd8-55f2400e3ee1","remote_addr":"[fd00:1122:3344:101::b]:44718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":323,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.616414783Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40600"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.616629358Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d3c3082a-ad53-448d-848a-80f084f6632d","remote_addr":"[fd00:1122:3344:101::b]:40600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":163,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.636712735Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37158"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.636816868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.674499236Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.724213458Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"07299cde-ccd5-4cfa-b11b-bdcbd700cdda","remote_addr":"[fd00:1122:3344:101::b]:37158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87437,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.724468494Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"be8b6b0a-4fe7-4349-95d2-e5745259724d","remote_addr":"[fd00:1122:3344:101::b]:37158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.757060427Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39420"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.75720708Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"75c6c1d9-057b-408b-a237-623fc71582ab","remote_addr":"[fd00:1122:3344:101::c]:39420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.779119449Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.77913528Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2e4cf77c-622e-48a0-97e8-145bab410b15","remote_addr":"[fd00:1122:3344:101::c]:39420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.884278015Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43392"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.884552841Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8e9585b1-3f0a-49c7-9bf6-ebeae6b3ccc4","remote_addr":"[fd00:1122:3344:101::c]:43392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":195,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.892575574Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"49a4de59-9ade-415d-8350-3c1894ae936c","remote_addr":"[fd00:1122:3344:101::c]:39420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":179,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.9911736Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41969"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:39.991266202Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:40.029496413Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:40.088998268Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"63a6ebf0-97ab-48ab-8fcf-25a58c4dc0c9","remote_addr":"[fd00:1122:3344:101::c]:41969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97759,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:40.089337446Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"67dd4125-30ca-445a-ab2e-0feb0b2e0a76","remote_addr":"[fd00:1122:3344:101::c]:41969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:40.437527567Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"b911036a-4ff1-4bbb-a643-f4c6f2b49d6d","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7778,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:58.952779794Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61993"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:57:58.952922847Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f5290105-74af-4e53-8fec-240f97c22217","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61993","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.625006977Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54594"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.62515835Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9bcdac0b-1287-4a7b-9726-e7f6d4592271","remote_addr":"[fd00:1122:3344:101::a]:54594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.679048548Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.679092419Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"988c9e8b-227b-4c0e-b6a9-cc01849d5850","remote_addr":"[fd00:1122:3344:101::a]:54594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":154,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.689643019Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b95427ed-760f-4e01-a585-94baedf770bb","remote_addr":"[fd00:1122:3344:101::a]:54594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.758736203Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58781"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.758915627Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d09f3260-4608-458a-b169-4ba9c3e2cd72","remote_addr":"[fd00:1122:3344:101::a]:58781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.770149973Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61148"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.770264956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.806301807Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.862617679Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a1f6feac-5d6e-4bbc-9c14-fc8df6110cca","remote_addr":"[fd00:1122:3344:101::a]:61148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92388,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:08.862887905Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c8968df1-a32a-4059-b617-208b0c942b07","remote_addr":"[fd00:1122:3344:101::a]:61148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.167263559Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46320"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.167416922Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cf31942c-f40c-46e8-bdae-b933371a4f26","remote_addr":"[fd00:1122:3344:101::b]:46320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":74,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.182824993Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.182842144Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"53472681-b27b-4ac0-8805-cd0c9fdf1912","remote_addr":"[fd00:1122:3344:101::b]:46320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.193154079Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9e506788-f528-46e8-ad2f-6c1623f0021c","remote_addr":"[fd00:1122:3344:101::b]:46320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":216,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.281129203Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36772"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.281333787Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"de8c0e2a-dd83-47af-ab80-77e39ea84eeb","remote_addr":"[fd00:1122:3344:101::b]:36772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":159,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.363773095Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59948"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.363877088Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.400067142Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.454715297Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0eff6155-0487-475c-928a-ae5bbdf188aa","remote_addr":"[fd00:1122:3344:101::b]:59948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90890,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.455084175Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5afe47cc-81a7-461c-98ae-39aa252c0197","remote_addr":"[fd00:1122:3344:101::b]:59948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.692476582Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56049"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.692592425Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3f47c4e4-3166-494c-aab0-838420d2571a","remote_addr":"[fd00:1122:3344:101::c]:56049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.734016849Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.734040429Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"31a807a4-f1bd-41c8-be62-008f8e97301e","remote_addr":"[fd00:1122:3344:101::c]:56049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.742966992Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cdbc593c-cdd6-4449-a9ec-6f6cba563723","remote_addr":"[fd00:1122:3344:101::c]:56049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.827036707Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56167"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.827225472Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"f524bcb1-4b25-42da-9683-be94be361187","remote_addr":"[fd00:1122:3344:101::c]:56167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.879230307Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42491"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.879311158Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.91714956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.972554962Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c9580522-e4e0-4110-87a9-764300e50662","remote_addr":"[fd00:1122:3344:101::c]:42491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93267,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:09.972932101Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"17f57c0a-0474-4605-b9d0-e66c59e7c7ff","remote_addr":"[fd00:1122:3344:101::c]:42491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:10.43645596Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"e53ffb4f-62f9-4da1-96f0-d20e9bcd1a61","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7482,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:28.988033524Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44476"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:28.988250119Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"8eb7814a-84de-4832-85e3-cb2721ed14f0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44476","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":44,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:38.690594667Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45664"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:38.690812372Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3f766915-4768-4e36-a842-e5da8fee2778","remote_addr":"[fd00:1122:3344:101::a]:45664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:38.717196204Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:38.717242565Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8b7e8e8b-9e8d-4b79-9542-d02933994697","remote_addr":"[fd00:1122:3344:101::a]:45664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:38.731120581Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"46853ca8-76ff-4bf4-b687-0c19747e00e6","remote_addr":"[fd00:1122:3344:101::a]:45664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":208,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.047055287Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36312"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.047266722Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d348f1a2-b259-4e2b-bdfa-6c79b4c32d07","remote_addr":"[fd00:1122:3344:101::a]:36312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":169,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.270978918Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54003"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.271165582Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.314405137Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.374354533Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"74cfee97-53ca-494a-901b-17b1c38893bd","remote_addr":"[fd00:1122:3344:101::a]:54003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103267,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.374634369Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"26c98625-9599-4e80-a2e2-4390eb98444e","remote_addr":"[fd00:1122:3344:101::a]:54003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.454593301Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49394"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.454753294Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e450e413-1af7-4387-905e-91dc35a07127","remote_addr":"[fd00:1122:3344:101::b]:49394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.627124811Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.627156911Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2a269e00-3a0c-40ee-bc80-c9c0279d3737","remote_addr":"[fd00:1122:3344:101::b]:49394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.642626984Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5e7c8567-930f-4559-ba19-94756fab02a8","remote_addr":"[fd00:1122:3344:101::b]:49394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":231,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.686232157Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:32901"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.68636508Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.732116562Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.795535257Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f79e27d7-4c2f-46a8-952e-0317b5bc59b8","remote_addr":"[fd00:1122:3344:101::b]:32901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109220,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.79567624Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58587"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.795889795Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d56bccbf-c898-4939-a831-d973ffacb8b0","remote_addr":"[fd00:1122:3344:101::b]:58587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.796031678Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c79ff0e2-758e-4a75-9d54-4366b33847c8","remote_addr":"[fd00:1122:3344:101::b]:32901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.852972905Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43508"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.853101018Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b5ab75c0-1f28-410c-9be5-cb91ca34303f","remote_addr":"[fd00:1122:3344:101::c]:43508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.867984227Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.868006908Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c000a7c0-f69d-4aa3-932c-890183be88a7","remote_addr":"[fd00:1122:3344:101::c]:43508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.906858223Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62493"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.907030647Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"fc569d25-63ca-4ce3-82ab-6a9ed66cc51b","remote_addr":"[fd00:1122:3344:101::c]:62493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:39.910877894Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b111ffbc-82a7-4bb5-b40f-01278ace8040","remote_addr":"[fd00:1122:3344:101::c]:43508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:40.177656481Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52880"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:40.177821235Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:40.215453432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:40.271920368Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"6063c33f-e405-4752-8c73-7b07bd2a52ee","remote_addr":"[fd00:1122:3344:101::c]:52880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94176,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:40.272244076Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fd179747-bd29-475e-8ec5-dae720398280","remote_addr":"[fd00:1122:3344:101::c]:52880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:40.436585489Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"0441393c-1e42-4d5e-979f-682be1f647fe","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7073,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:59.022564957Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46695"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:58:59.022716801Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"692eec89-b6e7-4ff8-90ac-abb4811efe3e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46695","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.681417125Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49410"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.681569868Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8831e56f-0952-4fc5-a062-59d1bb83508a","remote_addr":"[fd00:1122:3344:101::a]:49410","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.727321451Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.727350821Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"06799463-eb57-41c0-9a28-29ab91fbad9f","remote_addr":"[fd00:1122:3344:101::a]:49410","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.871615957Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54604"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.871799652Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8ae05879-e2c5-4b7c-aedb-5c8a5050f85c","remote_addr":"[fd00:1122:3344:101::a]:54604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.880868418Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7bbc5487-a73a-4c02-868e-abd321d6b12e","remote_addr":"[fd00:1122:3344:101::a]:49410","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":276,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.924119844Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55172"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.924193895Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:08.961181528Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.018666777Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"445582d3-4eb0-45d2-b9bc-3fd5272572b8","remote_addr":"[fd00:1122:3344:101::a]:55172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94499,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.019020065Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"19304db9-e48c-49e7-ac09-868bb8875825","remote_addr":"[fd00:1122:3344:101::a]:55172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.265178053Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36103"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.265336116Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"330a4a88-69db-4b55-9f7f-6241257cd64a","remote_addr":"[fd00:1122:3344:101::b]:36103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.304366755Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.304393196Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3c1abfda-e9d7-4d4a-bf0d-101eea899e89","remote_addr":"[fd00:1122:3344:101::b]:36103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.313022892Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ce9eb802-1098-454c-be0e-ec8401fb7016","remote_addr":"[fd00:1122:3344:101::b]:36103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":250,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.370987593Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46262"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.371259939Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"cf154175-7396-437a-8c3c-334ff97195c7","remote_addr":"[fd00:1122:3344:101::b]:46262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.38360582Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44373"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.383721693Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.421076624Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.480425886Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c0feb7e9-8f6d-49e8-8c7f-715d1e3fc6e8","remote_addr":"[fd00:1122:3344:101::b]:44373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96731,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.480779654Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c5272cc7-1dea-4095-92df-96309cf2e8bf","remote_addr":"[fd00:1122:3344:101::b]:44373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.602183739Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52452"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.602346753Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5d597aec-db9a-43a7-a18e-ffefeec84584","remote_addr":"[fd00:1122:3344:101::c]:52452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.656989108Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.657028968Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"bd2b2eb2-a483-46b4-bb4a-254f855f4eb9","remote_addr":"[fd00:1122:3344:101::c]:52452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.700879977Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:43694"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.701083262Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"fb4ab594-bb28-410c-a1a8-4c37eb7daa43","remote_addr":"[fd00:1122:3344:101::c]:43694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.704158692Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d3c0883f-8f26-447c-8c45-c3e119853109","remote_addr":"[fd00:1122:3344:101::c]:52452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.715707825Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:65460"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.715797587Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.749195428Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.80285906Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7e4943ca-b2ff-43a6-a55f-a83e6d9791ae","remote_addr":"[fd00:1122:3344:101::c]:65460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87082,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:09.803215509Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"014cdf78-e14f-4a76-99fc-c12152617889","remote_addr":"[fd00:1122:3344:101::c]:65460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:10.447958045Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"fa90eed9-3c65-4737-ada7-988a2b5e3971","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6094,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:22.887568785Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:29.056462556Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39590"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:29.056588809Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"c5a8797f-ac01-45b1-8be4-d13de1a65f54","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39590","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:38.750939175Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38956"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:38.751071578Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6103d1fc-b334-4225-80ee-e7beb1df0834","remote_addr":"[fd00:1122:3344:101::a]:38956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:38.815564768Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:38.815592658Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2c10b88c-b114-416a-b1ff-b1e7d034c1ee","remote_addr":"[fd00:1122:3344:101::a]:38956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:38.82400968Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"09c419a2-2485-4af3-9bde-01b0c9c7564f","remote_addr":"[fd00:1122:3344:101::a]:38956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":176,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.071758394Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52563"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.071988519Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c8f74437-57fb-4392-ae7f-332d883c4e2f","remote_addr":"[fd00:1122:3344:101::a]:52563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.165686543Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48634"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.165797265Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.20550987Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.260230737Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f6b68720-0895-4f73-8acf-53970696145e","remote_addr":"[fd00:1122:3344:101::a]:48634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94487,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.260571244Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3af4bb0d-6456-4b99-9d4c-fe59ae0c7fe0","remote_addr":"[fd00:1122:3344:101::a]:48634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.322097996Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39391"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.32229973Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a50bc605-4090-422c-9009-aae8aa65a8ca","remote_addr":"[fd00:1122:3344:101::b]:39391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.366695562Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.366741373Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f86a9675-644f-48a9-8d07-d1150f6234bd","remote_addr":"[fd00:1122:3344:101::b]:39391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.376888244Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"569ebecd-99e0-483e-bba4-f4ce736117fa","remote_addr":"[fd00:1122:3344:101::b]:39391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":351,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.470746342Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39221"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.470989277Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c39ede56-7ad1-4946-ad30-7775d1dfac4d","remote_addr":"[fd00:1122:3344:101::b]:39221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":196,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.545703379Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61165"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.545915754Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.584890552Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.648620994Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1bfa0906-d110-459c-9b9b-9b73bcb10042","remote_addr":"[fd00:1122:3344:101::b]:61165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102788,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.649153696Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d6947eb0-54dc-4333-a610-24ecfefd676b","remote_addr":"[fd00:1122:3344:101::b]:61165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":263,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.750787811Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57502"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.750947545Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e7b917a6-4bac-4c92-954c-b11aa2a85b28","remote_addr":"[fd00:1122:3344:101::c]:57502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.798386635Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.798419196Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"59f9c37c-7584-4d54-bcdb-9849a214d1d9","remote_addr":"[fd00:1122:3344:101::c]:57502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.808010764Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ab523933-3429-4752-9b59-14065221e080","remote_addr":"[fd00:1122:3344:101::c]:57502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":447,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.880378453Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48143"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.880542086Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1ad2d2b0-a8a4-4c06-b9fc-ec734de1ca54","remote_addr":"[fd00:1122:3344:101::c]:48143","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.927684771Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37444"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.927781753Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:39.965989763Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:40.021844705Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"711a4d09-473e-4376-8fab-5c9c429fa80e","remote_addr":"[fd00:1122:3344:101::c]:37444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94098,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:40.022143652Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d9ef7fef-d890-4ba1-8418-ccc31180bf1c","remote_addr":"[fd00:1122:3344:101::c]:37444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:40.531412612Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"bb594c00-aecd-42f4-9735-bc79a602fe37","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7248,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:59.091545152Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47406"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T17:59:59.091694065Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ab3cf4ce-33bb-4feb-9b27-7f09fbd43420","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47406","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.742807447Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60237"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.742980601Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"f68bb495-4135-4bee-9481-d84c1085f3e6","remote_addr":"[fd00:1122:3344:101::a]:60237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.761979733Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.762007844Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6f7e9d3a-ea35-4b7a-a463-7ef3d749fc68","remote_addr":"[fd00:1122:3344:101::a]:60237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.770948568Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"8f3f6a1b-d97b-4f5f-906c-a0e1d91babc8","remote_addr":"[fd00:1122:3344:101::a]:60237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":271,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.801299199Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63819"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.801458523Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d9ebc9de-e656-4694-b565-e312af04c47e","remote_addr":"[fd00:1122:3344:101::a]:63819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.813049537Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49735"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.813134919Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.849280082Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.90405347Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ddd55aed-760b-46e3-9f18-9576bc7de9e9","remote_addr":"[fd00:1122:3344:101::a]:49735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90949,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:08.904327916Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"455017c7-74fa-4790-9c5b-f75814da721d","remote_addr":"[fd00:1122:3344:101::a]:49735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.223421385Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48558"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.223575058Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"18b55269-b006-483b-92a3-6a45e8ba94cf","remote_addr":"[fd00:1122:3344:101::b]:48558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":82,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.275487851Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.275540722Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1bc339a1-bef7-4371-b36d-22ac8c2125e9","remote_addr":"[fd00:1122:3344:101::b]:48558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.285687823Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d9c49b68-04a5-48f9-a707-97f7d95939fa","remote_addr":"[fd00:1122:3344:101::b]:48558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":190,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.298757351Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62401"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.298830282Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.334975586Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.390396988Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"162d2938-f8bc-4973-9a24-dec0a3f03663","remote_addr":"[fd00:1122:3344:101::b]:62401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91589,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.39049221Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59322"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.390658164Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"2c611f00-a9bb-4197-bcc3-3b42fbfe4118","remote_addr":"[fd00:1122:3344:101::b]:59322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.390795037Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"857fa61f-9dc1-4539-a7e8-c107896eee8f","remote_addr":"[fd00:1122:3344:101::b]:62401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.607838151Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41096"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.607985164Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a11ef0ee-3a07-4903-9fde-3acf0c89a55e","remote_addr":"[fd00:1122:3344:101::c]:41096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.651513506Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.651543887Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"174b5c3e-6395-4daa-8ba8-6db47f53f8f2","remote_addr":"[fd00:1122:3344:101::c]:41096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.659447036Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4af260fb-813e-42f8-84e8-b2674a4d4019","remote_addr":"[fd00:1122:3344:101::c]:41096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":301,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.673150199Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53709"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.673259311Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.713224812Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.765710767Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"24a0a71c-2f98-4f15-8804-c654ad6e1d33","remote_addr":"[fd00:1122:3344:101::c]:53709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92491,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.765974783Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61271"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.76628901Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"04697ed3-2740-42fd-911d-eacd482df65b","remote_addr":"[fd00:1122:3344:101::c]:53709","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":265,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:09.766425204Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d4de901d-9d0a-42ad-b12e-a7411c8e4618","remote_addr":"[fd00:1122:3344:101::c]:61271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":258,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:10.533170949Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"e8925a35-56ce-47fb-9500-08a54afefa6b","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8800,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:29.134179449Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55720"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:29.134329323Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"33388903-e6dd-4f25-add1-93fcda3d73a4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55720","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:30.852117192Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51396"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:30.852373198Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7ba6f804-53d5-4f8b-91b7-b7aa59a2dae7","remote_addr":"[fd00:1122:3344:101::a]:51396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":184,"response_code":"200"}
{"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:30.853519604Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(13955958) }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:30.854789733Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"PUT","req_id":"1c8dd48a-3461-41bf-ad03-8001caee58d2","remote_addr":"[fd00:1122:3344:101::a]:51396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2202,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.028454259Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52743"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.032423029Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: d3065451-5d5f-40c9-9f95-bbb6046e4a73, project_id: 72f9ed17-3a48-492f-b279-5d4f818d9360 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-18T18:00:31.028187883Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(13955958))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-f090afa56c\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 056774e0-1053-486e-ae5c-a5b6f08b5185, read_only_parent: Some(Volume { block_size: 512, id: 16e8b41b-58e9-4f5a-bd14-98a2c8b2c23c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 474627a2-8d14-4ee0-8fde-fb4d378c6904, key: Some(\"HnZP228ope7abuWOUoIuif3eQr+bXrhCJvkmXNTKmSo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 056774e0-1053-486e-ae5c-a5b6f08b5185, key: Some(\"2mA5nYWeIWvwngwTQ6JQg1Eb11OsMmnkg1IZfaGG/qE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-f090afa56c\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.03248863Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.032499721Z","hostname":"buskin","pid":1572,"propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:1139","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(13955958))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 9377453b-88b2-4260-8fc5-8dfd3be4372b, kind: Instance { id: e7c1f70f-8643-4822-a729-69791d855790 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13955958), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-f090afa56c\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 056774e0-1053-486e-ae5c-a5b6f08b5185, read_only_parent: Some(Volume { block_size: 512, id: 16e8b41b-58e9-4f5a-bd14-98a2c8b2c23c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 474627a2-8d14-4ee0-8fde-fb4d378c6904, key: Some(\"HnZP228ope7abuWOUoIuif3eQr+bXrhCJvkmXNTKmSo=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 056774e0-1053-486e-ae5c-a5b6f08b5185, key: Some(\"2mA5nYWeIWvwngwTQ6JQg1Eb11OsMmnkg1IZfaGG/qE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-f090afa56c\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-18T18:00:31.028187883Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.034908666Z","hostname":"buskin","pid":1572,"uri":"/vmms/2bee446a-d317-4960-b95c-6357855030de","method":"PUT","req_id":"8cb0a2e3-1500-485e-824e-4d14ec4bfdd2","remote_addr":"[fd00:1122:3344:101::a]:52743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6347,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.146943788Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40657"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.147236764Z","hostname":"buskin","pid":1572,"uri":"/vmms/2bee446a-d317-4960-b95c-6357855030de/state","method":"PUT","req_id":"131fc7be-e5b8-4964-b770-eebf3e3b6db0","remote_addr":"[fd00:1122:3344:101::a]:40657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.148716748Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 13955958 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.148744739Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.190832428Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"}
{"msg":"Installing Omicron zone: oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.214894716Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.269550891Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60146"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.269746965Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b8d2331c-d7e7-401c-949e-cea5d6cc33bd","remote_addr":"[fd00:1122:3344:101::a]:60146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:31.290180791Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {9377453b-88b2-4260-8fc5-8dfd3be4372b: {10.151.1.176: {3d12df86-6d6b-46e5-b65a-0371b86694f6}, 10.151.1.175: {3d12df86-6d6b-46e5-b65a-0371b86694f6}}} }"}
{"msg":"Profile for oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value='oxControlInstance0'/>\n        <propval type=\"astring\" name=\"gateway\" value='fd00:1122:3344:101::1'/>\n        <propval type=\"astring\" name=\"listen_addr\" value='fd00:1122:3344:101::1:0'/>\n        <propval type=\"astring\" name=\"listen_port\" value='12400'/>\n        <propval type=\"astring\" name=\"metric_addr\" value='dns'/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:32.475997892Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Booting oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:32.476180086Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 40.124176ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:32.652332569Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 69.809172ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:32.701840696Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 187.780076ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:32.785623415Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 534.121321ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:32.991946385Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 607.815054ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:33.541421731Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 956.211316ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:34.161740822Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 1.425613843s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:35.128192876Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 529.144862ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:36.56564556Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\") failed: Property not found. retry in 1.393209594s","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:37.104628817Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.533870123Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:1633"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.545917568Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:1642"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.589785037Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:1666"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.731057605Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:835"}
{"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.732473207Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.732535659Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c783ef85-a35f-45e9-8dcb-26e6d95f3a32","remote_addr":"[fd00:1122:3344:101::a]:60146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7442607,"response_code":"204"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-10-18T18:00:38.733167593Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.733185444Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.733194294Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-18T18:00:38.733167593Z }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.733201654Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-18T18:00:38.733167593Z }, migration_in: None, migration_out: None }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.842171956Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57785"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.892410991Z","hostname":"buskin","pid":1572,"uri":"/vmms/2bee446a-d317-4960-b95c-6357855030de/state","method":"GET","req_id":"2d702cde-834a-4b5a-813a-b5517dce2c94","remote_addr":"[fd00:1122:3344:101::a]:57785","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":46186,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-10-18T18:00:38.893669079Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.89369136Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.89370723Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-18T18:00:38.893669079Z }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:38.893715811Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-18T18:00:38.893669079Z }, migration_in: None, migration_out: None }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.13031961Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39576"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.130508114Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.176630825Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.239979948Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c183e0d7-02fc-4379-9ced-fab1c00cec2c","remote_addr":"[fd00:1122:3344:101::a]:39576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109570,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.240179213Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62257"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.24051287Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1fff0ca4-f859-42e7-9521-6051cbbee147","remote_addr":"[fd00:1122:3344:101::a]:39576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":231,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.240697615Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"dad74132-38cf-4f2d-bc11-0560db65fd80","remote_addr":"[fd00:1122:3344:101::a]:62257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":289,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.241235687Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b7a01c3b-5135-4266-94ae-8e2e3b1fcdd2","remote_addr":"[fd00:1122:3344:101::a]:60146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":876,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.340121919Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58986"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.340250852Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"089cd258-4710-4389-b724-37bbd523988e","remote_addr":"[fd00:1122:3344:101::a]:58986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.385112804Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {9377453b-88b2-4260-8fc5-8dfd3be4372b: {10.151.1.176: {3d12df86-6d6b-46e5-b65a-0371b86694f6}, 10.151.1.175: {3d12df86-6d6b-46e5-b65a-0371b86694f6}}} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.386076446Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"779a148b-e2a9-4464-97c2-ab17c87ca129","remote_addr":"[fd00:1122:3344:101::a]:58986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1051,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.402130152Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"042edeb7-0152-4744-ba6a-8e2c513f8d13","remote_addr":"[fd00:1122:3344:101::a]:58986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":256,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.40689806Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39491"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.407048474Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9cebd55b-1c85-463f-97d0-2624f6cb61be","remote_addr":"[fd00:1122:3344:101::b]:39491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.484752314Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34469"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.488940919Z","hostname":"buskin","pid":1572,"uri":"/vmms/2bee446a-d317-4960-b95c-6357855030de/state","method":"GET","req_id":"92aa0bd1-26aa-445a-9d68-013318c87b8a","remote_addr":"[fd00:1122:3344:101::b]:34469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.565494823Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38243"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.565747089Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7955b5a6-0376-4e40-9b30-6e0bc6c468f7","remote_addr":"[fd00:1122:3344:101::b]:38243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.614378936Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {9377453b-88b2-4260-8fc5-8dfd3be4372b: {10.151.1.175: {3d12df86-6d6b-46e5-b65a-0371b86694f6}, 10.151.1.176: {3d12df86-6d6b-46e5-b65a-0371b86694f6}}} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.614408057Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d07e5d0b-6a41-42e5-9d16-311be2786ba3","remote_addr":"[fd00:1122:3344:101::b]:39491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.631310502Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d67686d9-eae3-48e7-88f0-75526c4af0d6","remote_addr":"[fd00:1122:3344:101::b]:39491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":251,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.690551482Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35312"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.690644914Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.733339016Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.796402763Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d952c38c-8ddb-4693-aa43-afca7d409d88","remote_addr":"[fd00:1122:3344:101::b]:35312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105783,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.796568487Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45486"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.796603647Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54374"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.796839443Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ab315f49-3631-45bc-a010-3673a4402221","remote_addr":"[fd00:1122:3344:101::c]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":137,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.796965716Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a9dd1c6b-f820-40ea-820b-870c0d45b969","remote_addr":"[fd00:1122:3344:101::b]:35312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":288,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.797069258Z","hostname":"buskin","pid":1572,"uri":"/vmms/2bee446a-d317-4960-b95c-6357855030de/state","method":"GET","req_id":"6173d1cc-8055-43dc-b8d8-215973c94d6f","remote_addr":"[fd00:1122:3344:101::c]:54374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":239,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.813662226Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {9377453b-88b2-4260-8fc5-8dfd3be4372b: {10.151.1.175: {3d12df86-6d6b-46e5-b65a-0371b86694f6}, 10.151.1.176: {3d12df86-6d6b-46e5-b65a-0371b86694f6}}} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.813696587Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"383503f6-a03a-4411-bd03-dc419fb1f12b","remote_addr":"[fd00:1122:3344:101::c]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":149,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.936068704Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:35248"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.936289319Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"08daaa7e-b9ff-4c2e-b170-5fa054df3ebc","remote_addr":"[fd00:1122:3344:101::c]:35248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:39.953758367Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"712ae181-0679-4a3c-b889-36a6dc9dd680","remote_addr":"[fd00:1122:3344:101::c]:45486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":802,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:40.024855677Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52088"}
{"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:40.024963489Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"}
{"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:40.061148324Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:40.114899208Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"08afc67d-c357-41bb-aed1-9745925109d1","remote_addr":"[fd00:1122:3344:101::c]:52088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89981,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:40.115335268Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7c92b2aa-cce6-4fe7-98df-78d1912c53b3","remote_addr":"[fd00:1122:3344:101::c]:52088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":227,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:40.532244824Z","hostname":"buskin","pid":1572,"uri":"/08066c3d-b7ed-49d2-8578-f8769fb94736","method":"GET","req_id":"e6e59a1b-3fc3-4d56-80c4-21246e069042","remote_addr":"[fd00:1122:3344:101::d]:37310","local_addr":"[fd00:1122:3344:101::1]:65402","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7607,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.671098749Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:46637"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.671898307Z","hostname":"buskin","pid":1572,"uri":"/vmms/2bee446a-d317-4960-b95c-6357855030de/state","method":"PUT","req_id":"8fb6b264-f36f-4739-81db-1e12cb607379","remote_addr":"[fd00:1122:3344:101::a]:46637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":680,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-10-18T18:00:48.672109261Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.672141012Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.672169113Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-18T18:00:48.672109261Z }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.672204334Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-18T18:00:48.672109261Z }, migration_in: None, migration_out: None }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-10-18T18:00:48.709757709Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.709777439Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.70980278Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-10-18T18:00:48.709757709Z }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.70981248Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:721","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.70982514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:48.709923943Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"4431706173448737589"}
{"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:48.709986384Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"14398483284864988566"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.712997683Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:48.713917724Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/62298c00-fb90-4e92-a4b4-95bca961978c/debug/bundle/zone\", \"/pool/int/e40afb95-7af8-4f62-aab4-e63b3bd02855/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/8853a1c1-d102-4f80-9012-356473e594ff/crypt/debug\", \"/pool/ext/8523f2c5-b045-4f20-8f22-89f361c6e33c/crypt/debug\", \"/pool/ext/24197587-83af-46c0-a36e-5e4a26350d21/crypt/debug\", \"/pool/ext/956d8649-5e02-4c83-a766-709d6c1c19f4/crypt/debug\", \"/pool/ext/4e3d3772-4d1a-4cd4-9451-d3f5402e0376/crypt/debug\", \"/pool/ext/aa3e117f-01a2-4173-8076-71ed9651de49/crypt/debug\", \"/pool/ext/e09cb849-5302-41dd-87a5-36e8723c48fb/crypt/debug\", \"/pool/ext/c6223f26-38f9-4483-9d17-efd8e60dd490/crypt/debug\", \"/pool/ext/883d1cbf-e1f8-4c86-b18f-d626feb923fd/crypt/debug\"] }","zone_name":"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:49.366455648Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de\", bundle_id: 031967eb-ca93-40f9-8a1a-b22d7b894461 }, time_created: 2024-10-18T18:00:48.714112748Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_2bee446a-d317-4960-b95c-6357855030de","v":0,"name":"SledAgent","level":40,"time":"2024-10-18T18:00:49.36655416Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:935"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.10720719Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.123687436Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(13955958), kind: System }"}
{"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.123728017Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(13955958), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.124110305Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-18T18:00:51.124082774Z }, migration_in: None, migration_out: None }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.146894654Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-18T18:00:51.124082774Z }, migration_in: None, migration_out: None }","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.150875635Z","hostname":"buskin","pid":1572,"instance_id":"e7c1f70f-8643-4822-a729-69791d855790","propolis_id":"2bee446a-d317-4960-b95c-6357855030de","instance_id":"e7c1f70f-8643-4822-a729-69791d855790","component":"InstanceManager","file":"sled-agent/src/instance.rs:872"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.599477603Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52505"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.599615457Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"651949d7-757d-4439-b6a9-ef03d7fa0781","remote_addr":"[fd00:1122:3344:101::a]:52505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.613925883Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {9377453b-88b2-4260-8fc5-8dfd3be4372b: {10.151.1.176: {3d12df86-6d6b-46e5-b65a-0371b86694f6}, 10.151.1.175: {3d12df86-6d6b-46e5-b65a-0371b86694f6}}} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.613950563Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"14319037-f41a-413c-b365-8e24cb528708","remote_addr":"[fd00:1122:3344:101::a]:52505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.622079748Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c12ef76d-7637-47fd-8f15-c8bc1e6fd268","remote_addr":"[fd00:1122:3344:101::a]:52505","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":262,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.730187451Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47812"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:51.730398166Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b6369476-aeec-4374-becd-df74c7d824a6","remote_addr":"[fd00:1122:3344:101::a]:47812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.206844879Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54993"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.207019433Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"508b3def-2a91-410f-8439-3041ba19f7b0","remote_addr":"[fd00:1122:3344:101::a]:54993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.295296493Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63711"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.295402956Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5c6d5bfd-81d4-4629-bbd7-90b613b46c85","remote_addr":"[fd00:1122:3344:101::a]:63711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.319064735Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.319147517Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.319155177Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b2bbc819-b7b9-4c15-aab8-44e095f048b9","remote_addr":"[fd00:1122:3344:101::a]:63711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:52.327815294Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c1f3dde6-c965-4107-9a5c-6dbe7c8167ee","remote_addr":"[fd00:1122:3344:101::a]:63711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":338,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:53.031593065Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34568"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:53.031694468Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"00af5926-491d-4d49-b990-82ea3650129e","remote_addr":"[fd00:1122:3344:101::a]:34568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"}
{"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:53.046346492Z","hostname":"buskin","pid":1572,"sled_id":"08066c3d-b7ed-49d2-8578-f8769fb94736","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:53.046369942Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"51de356b-aee9-4f7e-8930-f4d02544692c","remote_addr":"[fd00:1122:3344:101::a]:34568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":70,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-18T18:00:53.054208621Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"90af21f7-3db9-4a7f-a28e-040206fbf608","remote_addr":"[fd00:1122:3344:101::a]:34568","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":170,"response_code":"204"}