[ Dec 20 21:19:13 Enabled. ] [ Dec 20 21:19:13 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Dec 20 21:19:13 Method "start" exited with status 0. ] [ Dec 20 21:19:13 Rereading configuration. ] [ Dec 20 21:19:13 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:13.966967081Z","hostname":"buskin","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:13.967109914Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.020185755Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.020223785Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.134971498Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.143904739Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.151449938Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.750613654Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.888609878Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.888645969Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.888737751Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.888853083Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.888861814Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.888881574Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.891911812Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:14.891918842Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.891928983Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.891942753Z","hostname":"buskin","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.891971944Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.892046065Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.892118147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.892144657Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.892165558Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.903948922Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:14.903969922Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"648bb898-81e2-4bc5-92c0-fede81914617 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.002219195Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369133263Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369170173Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369177693Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369189904Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369203344Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369222985Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.369228365Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.37794172Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.37796083Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.463953929Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823814658Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823852039Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82388161Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82389017Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82390193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82390924Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823951451Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823957791Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823971082Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823975392Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823979522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.823995762Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824000283Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824012173Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824029123Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824032693Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824038053Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824042263Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824045373Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824051113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824055354Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824059474Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824079824Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824084204Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824087254Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824091434Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824096305Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824100345Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824116185Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824121105Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824123995Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824127335Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824150536Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824154826Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824160766Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824163846Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824167236Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824171656Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824195567Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824198837Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824210437Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824221537Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824268629Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824296429Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824301009Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82434458Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82434925Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82435362Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824356911Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824361251Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.82436456Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824368351Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824394951Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824401981Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824405592Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824416112Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824419512Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824443862Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824447642Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824452593Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824456013Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824509464Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824526694Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824535535Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824545375Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824552495Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824556605Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824584696Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.824593096Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.83548048Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:737"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.835519061Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999928218s","next_cleanup":"Instant { tv_sec: 1255, tv_nsec: 16369695 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.835552402Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.835575652Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.835635033Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.838467267Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.838858256Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.840551954Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841046825Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1203"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841053245Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841114306Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841144237Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841187238Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841200008Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841213028Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841219678Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841227039Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1252"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841231559Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841443403Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841578326Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1284"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841583017Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841586517Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841621578Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1316"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841678839Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841690709Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841840463Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.841971826Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.842041177Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8c74ff24-982c-40fd-86fd-ef04898c5922 (sled), rack_id: a7e68df9-95f5-4fda-aae0-2f46483e29ac, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.842998178Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4259"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.84306675Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4659"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.84754542Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.853654557Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.85376877Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.857920613Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.857983965Z","hostname":"buskin","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858068816Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:43397","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8c74ff24-982c-40fd-86fd-ef04898c5922 (sled), rack_id: a7e68df9-95f5-4fda-aae0-2f46483e29ac, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858192229Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.85820753Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.85824539Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.85825694Z","hostname":"buskin","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858262441Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858267991Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858308602Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858326912Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858338822Z","hostname":"buskin","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858345722Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858351853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858356073Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858387574Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.858396114Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.874533585Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.895486646Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.912545768Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.919939914Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:412"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.919990135Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.919998085Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:424"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.920029866Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 135.053976ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.923636347Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07c71896-9010-4be3-8cf9-cfdcf3206d38\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07c71896-9010-4be3-8cf9-cfdcf3206d38\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_648bb898-81e2-4bc5-92c0-fede81914617/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.933278403Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_648bb898-81e2-4bc5-92c0-fede81914617/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.955559062Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_648bb898-81e2-4bc5-92c0-fede81914617/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.955606874Z","hostname":"buskin","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.960915202Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:439"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.960925503Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_648bb898-81e2-4bc5-92c0-fede81914617/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:15.960930793Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.027656339Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.059724888Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 137.798859ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.060368043Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e485cae0-5c55-4a5e-a5cc-df54d1af21e5\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e485cae0-5c55-4a5e-a5cc-df54d1af21e5\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.199351179Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 165.329841ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.199863321Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d79f06b7-bef0-4e3a-91d2-73ceed933599\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d79f06b7-bef0-4e3a-91d2-73ceed933599\" }"} {"msg":"Mounting oxi_648bb898-81e2-4bc5-92c0-fede81914617/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.247296984Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.259690672Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.259773824Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.259795015Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.259827525Z","hostname":"buskin","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.366803624Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 227.327935ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.367978321Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"423e1a24-e83c-45f8-843d-f3bbe99021e0\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"423e1a24-e83c-45f8-843d-f3bbe99021e0\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.595722067Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 137.193751ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.596547896Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6543bcab-4cb5-4c91-a746-5ca5687aee54\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6543bcab-4cb5-4c91-a746-5ca5687aee54\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.735394049Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 184.409413ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.73586509Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f3c4712a-be30-411a-b78b-5ef335e38403\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f3c4712a-be30-411a-b78b-5ef335e38403\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.92092864Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 287.319931ms)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:16.92138157Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"58d7e1eb-284a-41a0-ae37-325536d99a1b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:19:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"58d7e1eb-284a-41a0-ae37-325536d99a1b\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.007374317Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.013228218Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.013266319Z","hostname":"buskin","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.013325211Z","hostname":"buskin","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:42379","producer_id":"8c74ff24-982c-40fd-86fd-ef04898c5922"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:17.014527508Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"144.127129ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.01553392Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.015540441Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.015611492Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:921","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.015625542Z","hostname":"buskin","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:17.160131223Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"474.515555ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:17.210123724Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:17.636693049Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.075547002s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:18.713508324Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.203674301s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.7024679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2683"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.702507081Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2928"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.702518471Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2553"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.702525331Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2600"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.702531701Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.702538552Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.704419694Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2853"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.705626021Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2594"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.705744184Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:19.705949808Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:19.918377141Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.664903958s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 52.59273ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:20.01023085Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 115.627727ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:20.229371574Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 283.159239ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:20.407762545Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 431.426653ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:20.749375975Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 932.922786ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:21.417980647Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.22697977s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:22.527499096Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:22.584679258Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"9.45020905s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 746.552124ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:23.86330025Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.326083371s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:24.793206631Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:26.610615973Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.640251737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.640727678Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642289293Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642312043Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642324164Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642340504Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642583569Z","hostname":"buskin","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642840155Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642871426Z","hostname":"buskin","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.642889026Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.831829293Z","hostname":"buskin","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:43397","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.831932905Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.831942536Z","hostname":"buskin","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.837126482Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4283","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.837145012Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4299"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:26.837163113Z","hostname":"buskin","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:27.065459271Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"499651769777759975"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:27.065584594Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8966795680073578518"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.092654481Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39994"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.093139692Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.093172662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.093196963Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.093216854Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.093378027Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"64e5f983-c0ff-44e5-ba33-5c400a0c374f","remote_addr":"[fd00:1122:3344:101::1]:39994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":400,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.123882251Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49260"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.123990744Z","hostname":"buskin","pid":1566,"uri":"/sled-role","method":"GET","req_id":"6b2d2a0f-3e83-4881-a0e4-71668acda2e3","remote_addr":"[fd00:1122:3344:101::1]:49260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: de6ab243-ff48-4380-b33f-9ab70458bfb5 (physical_disk), pool_id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 4bdf8b06-8a26-4967-865b-2e7497a90dff (physical_disk), pool_id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: a503cfc0-7ca5-4fa3-9a19-fa2511da2198 (physical_disk), pool_id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: f7060ad8-f0a6-4453-977a-a77e162dff3c (physical_disk), pool_id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 702e2fdd-9f41-451e-bbfa-d3cc9c6bd6c6 (physical_disk), pool_id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 630a26db-08a0-4667-9dec-da442cd2763c (physical_disk), pool_id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 20e6af2f-f702-40f7-b46d-cc224d45d0e4 (physical_disk), pool_id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: f5f695ac-4a5b-4701-b8a3-114cf7396d88 (physical_disk), pool_id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 33c33a95-6a94-47ef-ac93-2f0a308c23ef (physical_disk), pool_id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.1541487Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154232992Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:53882"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154557629Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: de6ab243-ff48-4380-b33f-9ab70458bfb5 (physical_disk), pool_id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 4bdf8b06-8a26-4967-865b-2e7497a90dff (physical_disk), pool_id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: a503cfc0-7ca5-4fa3-9a19-fa2511da2198 (physical_disk), pool_id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: f7060ad8-f0a6-4453-977a-a77e162dff3c (physical_disk), pool_id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 702e2fdd-9f41-451e-bbfa-d3cc9c6bd6c6 (physical_disk), pool_id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 630a26db-08a0-4667-9dec-da442cd2763c (physical_disk), pool_id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 20e6af2f-f702-40f7-b46d-cc224d45d0e4 (physical_disk), pool_id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: f5f695ac-4a5b-4701-b8a3-114cf7396d88 (physical_disk), pool_id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 33c33a95-6a94-47ef-ac93-2f0a308c23ef (physical_disk), pool_id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154647751Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154667651Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154675601Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154687722Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154724303Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154732993Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154740893Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.154747163Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.304724066Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.304747877Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.530839797Z","hostname":"buskin","pid":1566,"component":"VmmReservoirManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.623053134Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630728347Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630767567Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630783628Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630790848Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630796268Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630930461Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.630937241Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:27.643482073Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.665648619Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.673516486Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.805316802Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.813150907Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.813200758Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.813249099Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.813256689Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.81328581Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.823377376Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.823408627Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.915627125Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.92344516Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.923475821Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.923482181Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.923490131Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.923497721Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.923616844Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.923623784Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.959244153Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:27.966913175Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_fd3cb939-7a27-4242-9684-f71fce5b8238","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.105262537Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.113293468Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.113338789Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.113351179Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.113362929Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.113373689Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.122874842Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.122898513Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.214869435Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223410537Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223444707Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223453458Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223477698Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223515509Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223633882Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.223645462Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.259472215Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.26768569Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.406492812Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.414581764Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.414617644Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.414625005Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.414633175Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.414639395Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.424924796Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.424955196Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.516496069Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525120202Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525150633Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525160073Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525175613Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525187624Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525339477Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.525347417Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.561059938Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.56872911Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:28.646094385Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.703334008Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.710902008Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.710941169Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.710948299Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.710953949Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.71096798Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.720078444Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.720093194Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.80996255Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817659382Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817689593Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817695843Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817717933Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817728104Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817883747Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.817906878Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.852525914Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.860282818Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_dfc55466-0ce1-427c-805f-4acd9ba37666","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:28.993620327Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.00045172Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.000483191Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.000491601Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.000497101Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.000502881Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.011118329Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.01114458Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.109067706Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117332341Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117372262Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117381602Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117394583Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117405403Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117579017Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.117586157Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.151767753Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.159760833Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.300076079Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.308451987Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.308490608Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.308497528Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.308503818Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.308523578Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.318168005Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.318192995Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b7548ed7-f361-44b2-a96e-7c703af98968 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.411329214Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419305413Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419343863Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419351064Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419398235Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419408545Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419538198Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.419547728Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.455613837Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.463429382Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_b7548ed7-f361-44b2-a96e-7c703af98968","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.607244477Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.615713337Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.615738967Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.615743787Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.615749358Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.615753288Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.625976147Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.625991957Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"715f86b6-8047-4de8-af26-ede400e16cd7 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.72066931Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.728615458Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.72865915Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.72866725Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.72867814Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.72869402Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.728807723Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.728816173Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:29.732568977Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.76523627Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.773593747Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_715f86b6-8047-4de8-af26-ede400e16cd7","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.911140802Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.919181582Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.919213312Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.919222103Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.919229923Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.919235343Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.929453752Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:29.929492523Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.024664137Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032429111Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032495393Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032506363Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032519143Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032535993Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032667787Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.032675637Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.066972956Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.074540606Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.210362671Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217555242Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217608083Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217662355Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217774997Z","hostname":"buskin","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 648bb898-81e2-4bc5-92c0-fede81914617 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c4a1ae97-37ab-4b25-9caf-f353d764ffc2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217842059Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217854279Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.217921441Z","hostname":"buskin","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218168016Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:897"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218176026Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218183136Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218189216Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:928"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218197897Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:933"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218211897Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218221027Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:939"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218260498Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.218269388Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.21834067Z","hostname":"buskin","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"012c32a9-f03b-44e8-b134-b7332911b7f7","remote_addr":"[fd00:1122:3344:101::1]:53882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3064040,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {01fb7804-7247-4628-a9e0-0bd422a67cf1 (dataset): DatasetConfig { id: 01fb7804-7247-4628-a9e0-0bd422a67cf1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0387a60a-42f6-429d-b158-593c5e84ac0c (dataset): DatasetConfig { id: 0387a60a-42f6-429d-b158-593c5e84ac0c (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 07fcba39-2e1a-4e3b-91ee-cdcd81293afe (dataset): DatasetConfig { id: 07fcba39-2e1a-4e3b-91ee-cdcd81293afe (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09f58dde-bf37-4e39-851d-9322e28732f1 (dataset): DatasetConfig { id: 09f58dde-bf37-4e39-851d-9322e28732f1 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0f6b4ef6-dea6-4f09-81ec-afd3d0a5390c (dataset): DatasetConfig { id: 0f6b4ef6-dea6-4f09-81ec-afd3d0a5390c (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 17029657-2eea-44a8-8256-f21dab4248e1 (dataset): DatasetConfig { id: 17029657-2eea-44a8-8256-f21dab4248e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 18fbc52b-0d8d-43d7-adbe-6062cd625836 (dataset): DatasetConfig { id: 18fbc52b-0d8d-43d7-adbe-6062cd625836 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1982da06-6f78-40cb-88b4-3af4d26a1e99 (dataset): DatasetConfig { id: 1982da06-6f78-40cb-88b4-3af4d26a1e99 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 32cb5192-5206-47b3-85dc-a84461f574c6 (dataset): DatasetConfig { id: 32cb5192-5206-47b3-85dc-a84461f574c6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 34b0406d-6f77-4bda-954a-a4d5b4fcbcaa (dataset): DatasetConfig { id: 34b0406d-6f77-4bda-954a-a4d5b4fcbcaa (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3e616070-5a03-488b-ac8c-a47637402578 (dataset): DatasetConfig { id: 3e616070-5a03-488b-ac8c-a47637402578 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 3f0031d9-a0b1-4912-8fbe-3127c1d03e81 (dataset): DatasetConfig { id: 3f0031d9-a0b1-4912-8fbe-3127c1d03e81 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4285b250-6786-4be9-8a27-e126f0ebba4e (dataset): DatasetConfig { id: 4285b250-6786-4be9-8a27-e126f0ebba4e (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 437d6119-8ace-4d2b-bdbb-6e235283bdcf (dataset): DatasetConfig { id: 437d6119-8ace-4d2b-bdbb-6e235283bdcf (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 46288401-3a87-477b-b800-191c8b1cf64f (dataset): DatasetConfig { id: 46288401-3a87-477b-b800-191c8b1cf64f (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4665b51d-f9b7-41e2-a391-ace9ad04d8ab (dataset): DatasetConfig { id: 4665b51d-f9b7-41e2-a391-ace9ad04d8ab (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 47c6bcbf-e166-42ef-affe-ab48913f4f64 (dataset): DatasetConfig { id: 47c6bcbf-e166-42ef-affe-ab48913f4f64 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b556e59-13a7-4fe3-9650-d89ace1fbe6c (dataset): DatasetConfig { id: 4b556e59-13a7-4fe3-9650-d89ace1fbe6c (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 501df9fd-5739-44b9-82e0-e54b8b40c50d (dataset): DatasetConfig { id: 501df9fd-5739-44b9-82e0-e54b8b40c50d (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 51275b67-4af5-4f3a-95f4-6c1ef260b225 (dataset): DatasetConfig { id: 51275b67-4af5-4f3a-95f4-6c1ef260b225 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 55a5c0d6-8860-46e8-9fc8-969db897ae4a (dataset): DatasetConfig { id: 55a5c0d6-8860-46e8-9fc8-969db897ae4a (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5aac774d-c180-404e-b9ea-6b3066e7de5f (dataset): DatasetConfig { id: 5aac774d-c180-404e-b9ea-6b3066e7de5f (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5c29bfa1-e64a-406d-a08d-a3d5ede6b12e (dataset): DatasetConfig { id: 5c29bfa1-e64a-406d-a08d-a3d5ede6b12e (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5f376169-e6b1-49bc-8394-bf0490a5e762 (dataset): DatasetConfig { id: 5f376169-e6b1-49bc-8394-bf0490a5e762 (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6056bea8-a57f-4a4c-936c-c088652c5768 (dataset): DatasetConfig { id: 6056bea8-a57f-4a4c-936c-c088652c5768 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6064e394-c854-4040-a9c5-f0519761351a (dataset): DatasetConfig { id: 6064e394-c854-4040-a9c5-f0519761351a (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6643ba5e-cdc2-4974-8b61-c2a132ca9d4c (dataset): DatasetConfig { id: 6643ba5e-cdc2-4974-8b61-c2a132ca9d4c (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 69899e3f-2541-4b13-92b2-b88c0ef23f14 (dataset): DatasetConfig { id: 69899e3f-2541-4b13-92b2-b88c0ef23f14 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 6a7e75da-18de-438d-bf76-f9a44dfe8538 (dataset): DatasetConfig { id: 6a7e75da-18de-438d-bf76-f9a44dfe8538 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 7700a03d-7f5e-458f-84bf-777e1797b7f1 (dataset): DatasetConfig { id: 7700a03d-7f5e-458f-84bf-777e1797b7f1 (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 77965017-9beb-42d2-87c2-82c91ab49671 (dataset): DatasetConfig { id: 77965017-9beb-42d2-87c2-82c91ab49671 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7d57dc41-a2cd-4153-8923-281d619f3c33 (dataset): DatasetConfig { id: 7d57dc41-a2cd-4153-8923-281d619f3c33 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7d67ff24-4545-4212-a4fd-24c11710ae0b (dataset): DatasetConfig { id: 7d67ff24-4545-4212-a4fd-24c11710ae0b (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 80ce116e-30ba-4461-95cc-63c2b4e34448 (dataset): DatasetConfig { id: 80ce116e-30ba-4461-95cc-63c2b4e34448 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8f968a0d-9698-4aa4-9eb0-df4fec1100fc (dataset): DatasetConfig { id: 8f968a0d-9698-4aa4-9eb0-df4fec1100fc (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8fb26886-ae25-457f-8c28-81d5f60d890c (dataset): DatasetConfig { id: 8fb26886-ae25-457f-8c28-81d5f60d890c (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9054f396-5b84-40ed-93cf-5be0a257cd7a (dataset): DatasetConfig { id: 9054f396-5b84-40ed-93cf-5be0a257cd7a (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 96064b73-0270-4415-b656-a3f8d71c9a84 (dataset): DatasetConfig { id: 96064b73-0270-4415-b656-a3f8d71c9a84 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 98cb30fe-b8a3-4e45-96ca-1bf5decf5446 (dataset): DatasetConfig { id: 98cb30fe-b8a3-4e45-96ca-1bf5decf5446 (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9db61280-1e11-4abe-a007-6297ede28861 (dataset): DatasetConfig { id: 9db61280-1e11-4abe-a007-6297ede28861 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a0e0fcef-1213-4889-bc11-3f5740c90545 (dataset): DatasetConfig { id: a0e0fcef-1213-4889-bc11-3f5740c90545 (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a3cae6e6-2064-427c-8872-653d089c6702 (dataset): DatasetConfig { id: a3cae6e6-2064-427c-8872-653d089c6702 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aacfce92-5714-453f-a53b-ff1569ceef65 (dataset): DatasetConfig { id: aacfce92-5714-453f-a53b-ff1569ceef65 (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad229e7a-2b18-40e9-9c92-3bf4760a24a0 (dataset): DatasetConfig { id: ad229e7a-2b18-40e9-9c92-3bf4760a24a0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b4149ee2-3e78-4f3d-8018-4539304acd6b (dataset): DatasetConfig { id: b4149ee2-3e78-4f3d-8018-4539304acd6b (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b466a430-9dcf-4c44-89b7-fd9670b723be (dataset): DatasetConfig { id: b466a430-9dcf-4c44-89b7-fd9670b723be (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bd6135d1-34c1-429a-8a45-f649cb6760da (dataset): DatasetConfig { id: bd6135d1-34c1-429a-8a45-f649cb6760da (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c0738782-748f-4fb7-94ec-4e6a7e3996f4 (dataset): DatasetConfig { id: c0738782-748f-4fb7-94ec-4e6a7e3996f4 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c3c8efc6-919a-4bde-b667-8fe571efaf1c (dataset): DatasetConfig { id: c3c8efc6-919a-4bde-b667-8fe571efaf1c (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c5ec72af-2dd1-461c-a60f-8c1b7c9b10ad (dataset): DatasetConfig { id: c5ec72af-2dd1-461c-a60f-8c1b7c9b10ad (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c751860a-7f48-4cf9-95b3-1d98f7e923a8 (dataset): DatasetConfig { id: c751860a-7f48-4cf9-95b3-1d98f7e923a8 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cd8037d8-3817-4490-b821-62c4bd547676 (dataset): DatasetConfig { id: cd8037d8-3817-4490-b821-62c4bd547676 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d3b846a6-e155-4d87-8d98-250a5c1ef1ad (dataset): DatasetConfig { id: d3b846a6-e155-4d87-8d98-250a5c1ef1ad (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d4f6c34d-4e3c-4ec9-b898-e202dcf50452 (dataset): DatasetConfig { id: d4f6c34d-4e3c-4ec9-b898-e202dcf50452 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d79aa00f-de9a-4414-8572-ec141f23e8e1 (dataset): DatasetConfig { id: d79aa00f-de9a-4414-8572-ec141f23e8e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d851ae54-002b-407b-8b70-e2c3688b05a9 (dataset): DatasetConfig { id: d851ae54-002b-407b-8b70-e2c3688b05a9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, db554c04-0473-4a91-9dba-9c647e1c7bcb (dataset): DatasetConfig { id: db554c04-0473-4a91-9dba-9c647e1c7bcb (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e2ce4aad-ac49-4650-8ab1-70c104676c99 (dataset): DatasetConfig { id: e2ce4aad-ac49-4650-8ab1-70c104676c99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, edd63482-0a80-4a5a-a4d9-87db8ad35816 (dataset): DatasetConfig { id: edd63482-0a80-4a5a-a4d9-87db8ad35816 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, efa08c1b-4767-44f6-bccf-5c74b257339b (dataset): DatasetConfig { id: efa08c1b-4767-44f6-bccf-5c74b257339b (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f25c5dd3-47fd-4e16-a4c2-d8f858ad7949 (dataset): DatasetConfig { id: f25c5dd3-47fd-4e16-a4c2-d8f858ad7949 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f2c5d1f1-44f5-48fd-9b35-c34d809470b1 (dataset): DatasetConfig { id: f2c5d1f1-44f5-48fd-9b35-c34d809470b1 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f65b28b1-52bc-4f8b-b98e-1875db504109 (dataset): DatasetConfig { id: f65b28b1-52bc-4f8b-b98e-1875db504109 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f8c7ec1b-b3ee-405f-9e81-15de9538fcf7 (dataset): DatasetConfig { id: f8c7ec1b-b3ee-405f-9e81-15de9538fcf7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f92b104b-c43b-4060-8912-acb4e5aa9dc6 (dataset): DatasetConfig { id: f92b104b-c43b-4060-8912-acb4e5aa9dc6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ffaa5bd3-908d-469c-bbbf-6b115c160d70 (dataset): DatasetConfig { id: ffaa5bd3-908d-469c-bbbf-6b115c160d70 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250051911Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250189114Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42108"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250494471Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:859"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {01fb7804-7247-4628-a9e0-0bd422a67cf1 (dataset): DatasetConfig { id: 01fb7804-7247-4628-a9e0-0bd422a67cf1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0387a60a-42f6-429d-b158-593c5e84ac0c (dataset): DatasetConfig { id: 0387a60a-42f6-429d-b158-593c5e84ac0c (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 07fcba39-2e1a-4e3b-91ee-cdcd81293afe (dataset): DatasetConfig { id: 07fcba39-2e1a-4e3b-91ee-cdcd81293afe (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 09f58dde-bf37-4e39-851d-9322e28732f1 (dataset): DatasetConfig { id: 09f58dde-bf37-4e39-851d-9322e28732f1 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0f6b4ef6-dea6-4f09-81ec-afd3d0a5390c (dataset): DatasetConfig { id: 0f6b4ef6-dea6-4f09-81ec-afd3d0a5390c (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 17029657-2eea-44a8-8256-f21dab4248e1 (dataset): DatasetConfig { id: 17029657-2eea-44a8-8256-f21dab4248e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 18fbc52b-0d8d-43d7-adbe-6062cd625836 (dataset): DatasetConfig { id: 18fbc52b-0d8d-43d7-adbe-6062cd625836 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1982da06-6f78-40cb-88b4-3af4d26a1e99 (dataset): DatasetConfig { id: 1982da06-6f78-40cb-88b4-3af4d26a1e99 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 32cb5192-5206-47b3-85dc-a84461f574c6 (dataset): DatasetConfig { id: 32cb5192-5206-47b3-85dc-a84461f574c6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 34b0406d-6f77-4bda-954a-a4d5b4fcbcaa (dataset): DatasetConfig { id: 34b0406d-6f77-4bda-954a-a4d5b4fcbcaa (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3e616070-5a03-488b-ac8c-a47637402578 (dataset): DatasetConfig { id: 3e616070-5a03-488b-ac8c-a47637402578 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 3f0031d9-a0b1-4912-8fbe-3127c1d03e81 (dataset): DatasetConfig { id: 3f0031d9-a0b1-4912-8fbe-3127c1d03e81 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4285b250-6786-4be9-8a27-e126f0ebba4e (dataset): DatasetConfig { id: 4285b250-6786-4be9-8a27-e126f0ebba4e (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 437d6119-8ace-4d2b-bdbb-6e235283bdcf (dataset): DatasetConfig { id: 437d6119-8ace-4d2b-bdbb-6e235283bdcf (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 46288401-3a87-477b-b800-191c8b1cf64f (dataset): DatasetConfig { id: 46288401-3a87-477b-b800-191c8b1cf64f (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4665b51d-f9b7-41e2-a391-ace9ad04d8ab (dataset): DatasetConfig { id: 4665b51d-f9b7-41e2-a391-ace9ad04d8ab (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 47c6bcbf-e166-42ef-affe-ab48913f4f64 (dataset): DatasetConfig { id: 47c6bcbf-e166-42ef-affe-ab48913f4f64 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b556e59-13a7-4fe3-9650-d89ace1fbe6c (dataset): DatasetConfig { id: 4b556e59-13a7-4fe3-9650-d89ace1fbe6c (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 501df9fd-5739-44b9-82e0-e54b8b40c50d (dataset): DatasetConfig { id: 501df9fd-5739-44b9-82e0-e54b8b40c50d (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 51275b67-4af5-4f3a-95f4-6c1ef260b225 (dataset): DatasetConfig { id: 51275b67-4af5-4f3a-95f4-6c1ef260b225 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 55a5c0d6-8860-46e8-9fc8-969db897ae4a (dataset): DatasetConfig { id: 55a5c0d6-8860-46e8-9fc8-969db897ae4a (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5aac774d-c180-404e-b9ea-6b3066e7de5f (dataset): DatasetConfig { id: 5aac774d-c180-404e-b9ea-6b3066e7de5f (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5c29bfa1-e64a-406d-a08d-a3d5ede6b12e (dataset): DatasetConfig { id: 5c29bfa1-e64a-406d-a08d-a3d5ede6b12e (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5f376169-e6b1-49bc-8394-bf0490a5e762 (dataset): DatasetConfig { id: 5f376169-e6b1-49bc-8394-bf0490a5e762 (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6056bea8-a57f-4a4c-936c-c088652c5768 (dataset): DatasetConfig { id: 6056bea8-a57f-4a4c-936c-c088652c5768 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6064e394-c854-4040-a9c5-f0519761351a (dataset): DatasetConfig { id: 6064e394-c854-4040-a9c5-f0519761351a (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6643ba5e-cdc2-4974-8b61-c2a132ca9d4c (dataset): DatasetConfig { id: 6643ba5e-cdc2-4974-8b61-c2a132ca9d4c (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 69899e3f-2541-4b13-92b2-b88c0ef23f14 (dataset): DatasetConfig { id: 69899e3f-2541-4b13-92b2-b88c0ef23f14 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 6a7e75da-18de-438d-bf76-f9a44dfe8538 (dataset): DatasetConfig { id: 6a7e75da-18de-438d-bf76-f9a44dfe8538 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 7700a03d-7f5e-458f-84bf-777e1797b7f1 (dataset): DatasetConfig { id: 7700a03d-7f5e-458f-84bf-777e1797b7f1 (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 77965017-9beb-42d2-87c2-82c91ab49671 (dataset): DatasetConfig { id: 77965017-9beb-42d2-87c2-82c91ab49671 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7d57dc41-a2cd-4153-8923-281d619f3c33 (dataset): DatasetConfig { id: 7d57dc41-a2cd-4153-8923-281d619f3c33 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7d67ff24-4545-4212-a4fd-24c11710ae0b (dataset): DatasetConfig { id: 7d67ff24-4545-4212-a4fd-24c11710ae0b (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 80ce116e-30ba-4461-95cc-63c2b4e34448 (dataset): DatasetConfig { id: 80ce116e-30ba-4461-95cc-63c2b4e34448 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8f968a0d-9698-4aa4-9eb0-df4fec1100fc (dataset): DatasetConfig { id: 8f968a0d-9698-4aa4-9eb0-df4fec1100fc (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8fb26886-ae25-457f-8c28-81d5f60d890c (dataset): DatasetConfig { id: 8fb26886-ae25-457f-8c28-81d5f60d890c (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 9054f396-5b84-40ed-93cf-5be0a257cd7a (dataset): DatasetConfig { id: 9054f396-5b84-40ed-93cf-5be0a257cd7a (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 96064b73-0270-4415-b656-a3f8d71c9a84 (dataset): DatasetConfig { id: 96064b73-0270-4415-b656-a3f8d71c9a84 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 98cb30fe-b8a3-4e45-96ca-1bf5decf5446 (dataset): DatasetConfig { id: 98cb30fe-b8a3-4e45-96ca-1bf5decf5446 (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9db61280-1e11-4abe-a007-6297ede28861 (dataset): DatasetConfig { id: 9db61280-1e11-4abe-a007-6297ede28861 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, a0e0fcef-1213-4889-bc11-3f5740c90545 (dataset): DatasetConfig { id: a0e0fcef-1213-4889-bc11-3f5740c90545 (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a3cae6e6-2064-427c-8872-653d089c6702 (dataset): DatasetConfig { id: a3cae6e6-2064-427c-8872-653d089c6702 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, aacfce92-5714-453f-a53b-ff1569ceef65 (dataset): DatasetConfig { id: aacfce92-5714-453f-a53b-ff1569ceef65 (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ad229e7a-2b18-40e9-9c92-3bf4760a24a0 (dataset): DatasetConfig { id: ad229e7a-2b18-40e9-9c92-3bf4760a24a0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b4149ee2-3e78-4f3d-8018-4539304acd6b (dataset): DatasetConfig { id: b4149ee2-3e78-4f3d-8018-4539304acd6b (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b466a430-9dcf-4c44-89b7-fd9670b723be (dataset): DatasetConfig { id: b466a430-9dcf-4c44-89b7-fd9670b723be (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bd6135d1-34c1-429a-8a45-f649cb6760da (dataset): DatasetConfig { id: bd6135d1-34c1-429a-8a45-f649cb6760da (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c0738782-748f-4fb7-94ec-4e6a7e3996f4 (dataset): DatasetConfig { id: c0738782-748f-4fb7-94ec-4e6a7e3996f4 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c3c8efc6-919a-4bde-b667-8fe571efaf1c (dataset): DatasetConfig { id: c3c8efc6-919a-4bde-b667-8fe571efaf1c (dataset), name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c5ec72af-2dd1-461c-a60f-8c1b7c9b10ad (dataset): DatasetConfig { id: c5ec72af-2dd1-461c-a60f-8c1b7c9b10ad (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c751860a-7f48-4cf9-95b3-1d98f7e923a8 (dataset): DatasetConfig { id: c751860a-7f48-4cf9-95b3-1d98f7e923a8 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cd8037d8-3817-4490-b821-62c4bd547676 (dataset): DatasetConfig { id: cd8037d8-3817-4490-b821-62c4bd547676 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d3b846a6-e155-4d87-8d98-250a5c1ef1ad (dataset): DatasetConfig { id: d3b846a6-e155-4d87-8d98-250a5c1ef1ad (dataset), name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d4f6c34d-4e3c-4ec9-b898-e202dcf50452 (dataset): DatasetConfig { id: d4f6c34d-4e3c-4ec9-b898-e202dcf50452 (dataset), name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d79aa00f-de9a-4414-8572-ec141f23e8e1 (dataset): DatasetConfig { id: d79aa00f-de9a-4414-8572-ec141f23e8e1 (dataset), name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d851ae54-002b-407b-8b70-e2c3688b05a9 (dataset): DatasetConfig { id: d851ae54-002b-407b-8b70-e2c3688b05a9 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, db554c04-0473-4a91-9dba-9c647e1c7bcb (dataset): DatasetConfig { id: db554c04-0473-4a91-9dba-9c647e1c7bcb (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e2ce4aad-ac49-4650-8ab1-70c104676c99 (dataset): DatasetConfig { id: e2ce4aad-ac49-4650-8ab1-70c104676c99 (dataset), name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, edd63482-0a80-4a5a-a4d9-87db8ad35816 (dataset): DatasetConfig { id: edd63482-0a80-4a5a-a4d9-87db8ad35816 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, efa08c1b-4767-44f6-bccf-5c74b257339b (dataset): DatasetConfig { id: efa08c1b-4767-44f6-bccf-5c74b257339b (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f25c5dd3-47fd-4e16-a4c2-d8f858ad7949 (dataset): DatasetConfig { id: f25c5dd3-47fd-4e16-a4c2-d8f858ad7949 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f2c5d1f1-44f5-48fd-9b35-c34d809470b1 (dataset): DatasetConfig { id: f2c5d1f1-44f5-48fd-9b35-c34d809470b1 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f65b28b1-52bc-4f8b-b98e-1875db504109 (dataset): DatasetConfig { id: f65b28b1-52bc-4f8b-b98e-1875db504109 (dataset), name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f8c7ec1b-b3ee-405f-9e81-15de9538fcf7 (dataset): DatasetConfig { id: f8c7ec1b-b3ee-405f-9e81-15de9538fcf7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f92b104b-c43b-4060-8912-acb4e5aa9dc6 (dataset): DatasetConfig { id: f92b104b-c43b-4060-8912-acb4e5aa9dc6 (dataset), name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ffaa5bd3-908d-469c-bbbf-6b115c160d70 (dataset): DatasetConfig { id: ffaa5bd3-908d-469c-bbbf-6b115c160d70 (dataset), name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250621844Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250626354Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250634924Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.250638364Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.26651717Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.266542591Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.266546951Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.326241519Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.32627335Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.32627845Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\"), full_name: \"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.385883327Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.385912697Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.385916598Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\"), full_name: \"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.446559668Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.446588708Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.446593308Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.497383767Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.497414828Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.497420698Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.545506536Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.545541637Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.545546387Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.595347254Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.595378575Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.595393215Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.655213096Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.655240497Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.655244257Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.715239512Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.715270463Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.715274103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:30.734403932Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.763549046Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.763583266Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.763593367Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.813305781Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.813333702Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.813337652Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/debug\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.843483648Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.843513309Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.843517899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.892831215Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.892859015Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.892863035Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.954299453Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.954328084Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.954331754Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/debug\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.984427098Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.984452358Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:30.984456269Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/715f86b6-8047-4de8-af26-ede400e16cd7/crypt/debug\"), full_name: \"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.013672264Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.013700194Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.013705594Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.062708703Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.062739934Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.062743914Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.124690893Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.124713474Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.124717194Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.173637331Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.173660411Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.173664271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.233405671Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.233437002Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.233441842Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.282637365Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.282674056Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.282678216Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.311800249Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.311824639Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.311828219Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.341141657Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.341169277Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.341173717Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.400374655Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.400404916Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.400409656Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone\"), full_name: \"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.429994409Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.43002112Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.430025Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.481399482Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.481426482Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.481430333Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.529621023Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.529647834Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.529651684Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.590485788Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.590510639Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.590524529Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/debug\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.618425715Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.618451855Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.618456755Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/debug\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.64587178Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.64589245Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.645896041Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b7548ed7-f361-44b2-a96e-7c703af98968/crypt/debug\"), full_name: \"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.673597702Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.673621872Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.673626143Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.732426941Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.732456972Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.732461852Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:31.736659316Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.79082394Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.790852231Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.790857291Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.847776268Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.847800558Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.847804458Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.905602354Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.905631425Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.905635305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.954784997Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.954812178Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.954815958Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/debug\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.984195547Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.984217007Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:31.984220747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:32.036278104Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"22.575048353s"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.048211412Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.048221642Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.048225522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.101047577Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.101073577Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.101078117Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\"), full_name: \"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone/oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.160534491Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.160563461Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.160568331Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.207968685Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.207991315Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.208006085Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/debug\"), full_name: \"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.237876195Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.237903246Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.237907806Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.268618525Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.268643795Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.268647505Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\"), full_name: \"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.329293155Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.329324986Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.329329706Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.376004122Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.376041563Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.376045833Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\"), full_name: \"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone/oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.436845937Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone/oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.436873317Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone/oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.436877947Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone/oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\"), full_name: \"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone/oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.494728475Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.494760375Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.494765056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.541235007Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.541262358Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.541278979Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.588477877Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.588521888Z","hostname":"buskin","pid":1566,"name":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.588529168Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone\"), full_name: \"oxp_715f86b6-8047-4de8-af26-ede400e16cd7/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.617931137Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.617957778Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.617962728Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone\"), full_name: \"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.646559459Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.64658692Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.64659062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.674792802Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.674823913Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.674835863Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/zone/oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.736293172Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone/oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.736321202Z","hostname":"buskin","pid":1566,"name":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone/oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.736325002Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone/oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\"), full_name: \"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/zone/oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:32.73980619Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.795395697Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.795424787Z","hostname":"buskin","pid":1566,"name":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.795429558Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.841088711Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.841115482Z","hostname":"buskin","pid":1566,"name":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.841131302Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\"), full_name: \"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/zone/oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.899858579Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.89988532Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.89988923Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.960727534Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.960761765Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:32.960766635Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.019732947Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.019758857Z","hostname":"buskin","pid":1566,"name":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.019762547Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/debug\"), full_name: \"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.048842659Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.0488675Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.04887177Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.107915554Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.107949135Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.107954665Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/zone/oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.166062138Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.166093569Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.166098449Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone/oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.224511088Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.224543999Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.224548779Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/zone/oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.286939388Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.286966749Z","hostname":"buskin","pid":1566,"name":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.286970569Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.336536131Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.336565271Z","hostname":"buskin","pid":1566,"name":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.336570251Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone\"), full_name: \"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.366361099Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.36638592Z","hostname":"buskin","pid":1566,"name":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.36639554Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone\"), full_name: \"oxp_b7548ed7-f361-44b2-a96e-7c703af98968/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.395645136Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.395670266Z","hostname":"buskin","pid":1566,"name":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.395674747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.443192702Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.443512739Z","hostname":"buskin","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.443746985Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.443846377Z","hostname":"buskin","pid":1566,"uri":"/datasets","method":"PUT","req_id":"e13ec4b7-9974-4d36-8b77-eed35b41307c","remote_addr":"[fd00:1122:3344:101::1]:42108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3193597,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.47610547Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.476239573Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57945"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.47655044Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.47656014Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.484399096Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.484419307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.493312596Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.493343857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.50196352Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.501997931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.502010511Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.502087463Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.544008843Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.544037064Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.544041084Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.600745655Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.604252424Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.607888625Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.620256703Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.624647331Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:33.628232912Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8c74ff24-982c-40fd-86fd-ef04898c5922)","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:33.742673228Z","hostname":"buskin","pid":1566,"component":"ProbeManager","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:35.194020031Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:35.194116183Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:35.674684979Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:35.67473398Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:35.67831095Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.086817581Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4307"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.111056314Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4317","gateway":"fd00:1122:3344:101::1"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.116603919Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4331","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.116616579Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4396"} {"msg":"Profile for oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.160128215Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.160189056Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 62.830246ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.183888307Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.183935519Z","hostname":"buskin","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.184035921Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 129.499436ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.257290353Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.313567095Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4462"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.313627337Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4489"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 51.542821ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.317692558Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 49.043443ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.324102682Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.355220079Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4502"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.35525576Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4348"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 77.013933ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.377023358Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 120.109865ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.385292724Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 208.695778ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.489101971Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 106.015518ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.515917823Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 288.210622ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.527943213Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.528021044Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4390"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.528037495Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4469","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.618119635Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4480"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.618270518Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4522"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:36.621207724Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3448911829135188077"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:36.62193875Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8343298992670031128"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 568.934076ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.654977021Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 469.519917ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.72674231Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.802750905Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4554"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.80922646Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4560"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 544.790528ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:36.831015178Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.927507212Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4601"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.927566784Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.962945587Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.995546057Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.99565748Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.996190252Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:36.996208952Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.033182811Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:37.067876419Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"499651769777759975"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:37.072068803Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8966795680073578518"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089227808Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089267699Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089273689Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089290249Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089296229Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.08930238Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.08930757Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.08931192Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089318Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.08932306Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089339241Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089345631Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089350051Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089353681Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089359131Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089363691Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089368011Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089372181Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089376451Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089380781Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089384982Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089388322Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:37.089401032Z","hostname":"buskin","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 1.11058131s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:37.206572819Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 1.151434224s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:37.227336425Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 527.911937ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:37.385071202Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 995.505784ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:37.92370237Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 1.426603146s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:38.32726692Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 1.471096338s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:38.390711662Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 1.48658876s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:38.927828036Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 872.290924ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:39.763369762Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 628.988468ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:39.870511434Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 617.643944ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:40.423294849Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc\") failed: Property not found. retry in 773.327955ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:40.508485929Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37\") failed: Property not found. retry in 569.270506ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:40.644617482Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe\") failed: Property not found. retry in 657.99049ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:41.049883969Z","hostname":"buskin","pid":1566,"zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.72911847Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_4565f09e-ee63-469d-bb6e-3ccd4abd01fe"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.729156941Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_73abfaca-4f35-4f97-9980-1890d6dba8fc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.729163351Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_f3f6b067-d10e-44b3-9f21-f8e5a96a2b37"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.729174001Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.729456557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.729722473Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"1cdf5eaa-e9a0-43e9-82b8-91b034c04e22","remote_addr":"[fd00:1122:3344:101::1]:57945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8253355,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:41.7318005Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 132.60943ms)","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:41.998190702Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.151760936Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 56.231165ms)","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:42.181872031Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.256188488Z","hostname":"buskin","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.302593679Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.302607509Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.335295912Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.336049139Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.336054309Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.365815296Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.365919948Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:36005"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.366086202Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.366091882Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.376417244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.376440044Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.385717992Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.385750623Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.395580024Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.395608734Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.395772138Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.39587906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.40655417Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.437941743Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.438580688Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.438592058Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.468293394Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 228, 3]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.468306634Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.468799875Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.468809216Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.469092182Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.470847721Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.493881008Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:42.51137723Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:43.124908228Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:43.125078042Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 38.693555ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:43.262935813Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 66.36998ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:43.309438886Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 191.580473ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:43.387069406Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 362.940719ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:43.592048923Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 500.22364ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:43.964375362Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 1.042285275s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:44.474665534Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 1.255958807s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:45.525083758Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:46.623048738Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3448911829135188077"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:46.623183371Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8343298992670031128"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 1.222137481s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:46.790533144Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:47.072405474Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"499651769777759975"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:47.073535199Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8966795680073578518"} {"msg":"wait for service Some(\"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\") failed: Property not found. retry in 781.314455ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:48.02261712Z","hostname":"buskin","pid":1566,"zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.822043276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.822128598Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.822469516Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.822799973Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"191fb1fe-a9b8-436e-9649-e1c90f989d82","remote_addr":"[fd00:1122:3344:101::1]:36005","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6456825,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.824964322Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.855076257Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.855243621Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:47335"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:48.864028438Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:48.86412021Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"d53ef094-3aed-4fc0-ad29-1e52920a5d37","remote_addr":"[fd00:1122:3344:101::1]:47335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8652,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:48.866214647Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:49.049156649Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:49.049325143Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:34329"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:51.239680979Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"15804955486547187899"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:51.30797198Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18342201734782093595"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:51.732400877Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"14638909120809600788"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:53.059156977Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:53.05929684Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"382d9ed7-f234-41eb-970c-6ad37f320ac8","remote_addr":"[fd00:1122:3344:101::1]:34329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009861,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:53.061404507Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:53.261020183Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:53.261158366Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:58791"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:53.269987354Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:53.270169658Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"8027e0a5-cb57-4c19-a269-2efec932ceab","remote_addr":"[fd00:1122:3344:101::1]:58791","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8838,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:53.272875879Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:53.459448443Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:53.459616966Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46041"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:56.228547749Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3448911829135188077"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:56.22866267Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8343298992670031128"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:56.676849577Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"499651769777759975"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:56.677995673Z","hostname":"buskin","pid":1566,"component":"kstat-sampler-worker","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"8966795680073578518"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b6c9cbef-5cb0-4145-b5d4-2d3d002e7390': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-12-20T21:19:57.068477955Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.068587678Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"90c9a223-367a-4c48-8586-a24b3c83feae","remote_addr":"[fd00:1122:3344:101::1]:46041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008874,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:19:57.070398868Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.204207799Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.204346872Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:57808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.225478596Z","hostname":"buskin","pid":1566,"uri":"/timesync","method":"GET","req_id":"103e409d-178e-4ab6-b7b1-12e139caa928","remote_addr":"[fd00:1122:3344:101::1]:57808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21044,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1734729596.4963584, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.228213607Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.228222807Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1392"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.258328642Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.258439715Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42131"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.25869377Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.258699091Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.269512993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.269537164Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.278986885Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.279009486Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.289026151Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.289056181Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.298451872Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.298487973Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.30725639Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.30728223Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.315828182Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.315851552Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.324752172Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.324777902Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.3335674Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.33359307Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.333773234Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.333875636Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.415532767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.415567188Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.415573788Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.415578028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.415584108Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.506696891Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.51106382Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.515267224Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.519406996Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.523772885Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.528311126Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.532660814Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.537028842Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.541474811Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:19:57.546104025Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.505410794Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.505537716Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.515177739Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.515287616Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.584529747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.584645437Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.642558266Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.642662378Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 53.312836ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:00.656778745Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.657036188Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:00.657132537Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 47.534959ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:00.675973375Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 76.644463ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:00.720503703Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 45.468055ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:00.736374483Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 106.877742ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:00.74712609Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 170.355833ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.296021019Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 144.5951ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.416433293Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 234.509277ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.450388236Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 63.11993ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.467800681Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 296.386948ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.482396362Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 54.153006ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.495185169Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 52.550721ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.547535591Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 112.299718ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.67797619Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 149.765855ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.685376598Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 169.836572ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:01.720493803Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 514.819078ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.04389443Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 162.084188ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.083658494Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 556.176931ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.104644759Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 1.194174298s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.118573097Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 217.890427ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.1360257Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 431.942592ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.258823552Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 831.8285ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.370880225Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 477.619506ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.57159285Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 651.224486ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.674922007Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 546.19328ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:02.70423071Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 847.91789ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:03.064804276Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 1.010521863s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:03.21757713Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 878.336857ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:03.262475665Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 953.172286ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:03.327718393Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 1.068595134s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:03.338147969Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 1.492144865s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:03.925818024Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 1.168978506s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:04.152357708Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 937.901741ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:04.242003473Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 839.641598ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:04.29237146Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 828.818438ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:04.418952413Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 1.413213542s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:05.145160359Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 1.23661793s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:05.192178785Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 664.946921ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:05.260780803Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 1.159262036s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:05.333559734Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 587.333403ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:05.432413423Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 1.212361039s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:05.938579338Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:05.966575292Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:05.966834911Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b000e6e7-e69c-4435-a341-fa41e0c21ce3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42224","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 1.072271789s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:06.034047688Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 1.137012616s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:06.442002365Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 1.46958833s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:06.505206582Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 984.490971ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:06.579038695Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 1.397504s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:07.121185158Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 712.912498ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:07.164918268Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 1.397189261s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:07.577416216Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 805.044653ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:07.591969864Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 895.120376ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:07.888848116Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 636.34588ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:07.986633246Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 850.018207ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:08.409500958Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 1.389544618s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:08.531871929Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 1.092503271s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:08.633458472Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 1.145939522s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:08.796098193Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 998.617285ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:08.987342492Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 674.87262ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.271678405Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.276970541Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"139.440785ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 809.470226ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.73966919Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d\") failed: Property not found. retry in 1.123398162s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.935836605Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980\") failed: Property not found. retry in 1.351519582s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.953217775Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97\") failed: Property not found. retry in 882.755659ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.960469112Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00\") failed: Property not found. retry in 1.380753077s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:09.997220632Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910\") failed: Property not found. retry in 842.118079ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:10.560859992Z","hostname":"buskin","pid":1566,"zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.425638547Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_50bfb30f-8e0c-4b57-a131-782097c61910"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.425759568Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.425767274Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_4939e8c6-7a63-45f9-a24a-89f71cba5b00"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.425772459Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_ad844bf5-de77-449c-b06f-50b2df3f4d97"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.425777596Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_dbf50d9a-906f-459e-a09e-f219a42e264d"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.4258011Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.426206372Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.426551698Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"856d9730-b1f7-48fd-8ac1-454844edc4f6","remote_addr":"[fd00:1122:3344:101::1]:42131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":14134298,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.466780289Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:47627"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.49738451Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"f54a9cb9-0066-4cd3-8398-9846dbd16d8c","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":30344,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:11.497611475Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"72.516581ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f54a9cb9-0066-4cd3-8398-9846dbd16d8c\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f54a9cb9-0066-4cd3-8398-9846dbd16d8c\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.604919865Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"3ff8ab7f-3053-4c08-872c-a0f01f120742","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":33741,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:11.605120214Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"106.181584ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3ff8ab7f-3053-4c08-872c-a0f01f120742\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3ff8ab7f-3053-4c08-872c-a0f01f120742\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.74128689Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"63fa1239-69ec-4665-86bd-2005b5c7fe1e","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28584,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:11.74146097Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"107.497223ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63fa1239-69ec-4665-86bd-2005b5c7fe1e\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63fa1239-69ec-4665-86bd-2005b5c7fe1e\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:11.878857798Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"4bd3cdcd-d64c-4de7-a6f7-a235549aff59","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28802,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:11.879071009Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"206.502148ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4bd3cdcd-d64c-4de7-a6f7-a235549aff59\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4bd3cdcd-d64c-4de7-a6f7-a235549aff59\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:12.115729678Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"102b3994-9ae1-4a54-87df-fb05c135a22b","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27564,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:12.115901779Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"104.76458ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"102b3994-9ae1-4a54-87df-fb05c135a22b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"102b3994-9ae1-4a54-87df-fb05c135a22b\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:12.250433026Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"201eb69d-1f61-4856-90c3-722bc918e7aa","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27989,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:12.250599173Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"352.816079ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"201eb69d-1f61-4856-90c3-722bc918e7aa\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"201eb69d-1f61-4856-90c3-722bc918e7aa\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:12.633271891Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"1bf82220-70a8-4193-b630-77fe74a38df1","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_4728b7fd-e948-4a22-a906-8df24c162980 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27195,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:12.633440636Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1157","retry_after":"240.793552ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1bf82220-70a8-4193-b630-77fe74a38df1\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1bf82220-70a8-4193-b630-77fe74a38df1\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:12.91193772Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3758"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:33.263887833Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3776"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.899609623Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.90087626Z","hostname":"buskin","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"cdbab794-236d-4b6e-9398-771eaead45bd","remote_addr":"[fd00:1122:3344:101::1]:47627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32936228,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.902643054Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.904291322Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6710dab9-b6d9-45cd-b145-6bdf333b3948","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42192","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":455,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:45.90905623Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"525.475224ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.944729316Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.945397384Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:52963"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.948043098Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.948123585Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.962658993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.962707088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.973720795Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.97376064Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.983336938Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.983373886Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.993495782Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:45.993531005Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.003137982Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.003181183Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.012728768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.012761844Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.021975103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.022004495Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.032422844Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.032454186Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.045000263Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.045039179Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.05586357Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.055898915Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.068133152Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.068163967Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.076514232Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.076547215Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.087911944Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.087942227Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.097327381Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.097363128Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.107193517Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.107233305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.116689075Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.116720988Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.125618942Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.1256544Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.145685662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.145727185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.159825164Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.159857537Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.172012244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.172049661Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.172564331Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.172907694Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303543152Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303582275Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303586972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.30359075Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303594678Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303598419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303601953Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303606031Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303611656Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303616964Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303620751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303624351Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303628054Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303641601Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303645304Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303649101Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303652468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303655955Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.303659405Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.303826134Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9d32e4cf-54d6-4571-bba2-8552740e069b","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized)]","latency_us":357174,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.304397851Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"118.365087ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d32e4cf-54d6-4571-bba2-8552740e069b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d32e4cf-54d6-4571-bba2-8552740e069b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.424899554Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.425228371Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.425234436Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.436889286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.43692794Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.447304715Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.4473357Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.459165408Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.459207343Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.470148889Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.470195708Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.480874265Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.480918338Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.491108664Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.491139415Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.500645097Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.500678473Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.510391562Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.510422341Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.520033235Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.520084555Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.529207613Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.529239442Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.539354447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.539388722Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.54900798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.549041093Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.55860833Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.558645295Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.568345009Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.568387019Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.577866054Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.577898614Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.587318728Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.587351842Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.598308973Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.598346286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.60830276Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.608340036Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.617734032Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.617777233Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.627453042Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.627503209Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.627736557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.627874702Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725699767Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725751771Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725759281Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725764588Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725770869Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725775941Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725781969Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725786844Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725790557Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725794072Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725804572Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725809691Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725813123Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725818289Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725823061Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725827645Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725831001Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.725834236Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.72583747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.725879134Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"1f25e955-5210-4131-88bb-8f42cbc5d677","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized)]","latency_us":299993,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:46.726031369Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"120.15438ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f25e955-5210-4131-88bb-8f42cbc5d677\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1f25e955-5210-4131-88bb-8f42cbc5d677\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.847656525Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.847941241Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.847953747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.859847924Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.859882508Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.870432301Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.870459049Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.881497074Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.881534555Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.890746041Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.890773398Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.900064478Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.900089266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.9097258Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.909751339Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.920389705Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.920414746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.932134478Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.932163486Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.943010039Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.943038633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.952929654Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.952957919Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.96319494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.963218368Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.973031128Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.973054275Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.982548804Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.982581767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.992419878Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:46.992444412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.001714372Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.001736094Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.011061117Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.011088709Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.021386126Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.021408392Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.030776804Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.030802407Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.0397774Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.039801869Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.049285914Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.049320078Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.049499397Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.049626056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146148189Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146188502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146194258Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146199255Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.14620443Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146208171Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146211594Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146228525Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.14623325Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146237778Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146242794Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.1462484Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146253463Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146258234Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146264873Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.14626971Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146274679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146278026Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146282704Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.146423759Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9c72a895-e011-46b6-9bec-7827a1310fe0","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized)]","latency_us":297840,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.146574568Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"215.249673ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c72a895-e011-46b6-9bec-7827a1310fe0\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9c72a895-e011-46b6-9bec-7827a1310fe0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.363569124Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.363824795Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.36383222Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.375328202Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.375366959Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.386648414Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.386686412Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.397876881Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.39791277Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.408001836Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.408101906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.417926497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.417959976Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.42828649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.428322105Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.438288862Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.438330769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.447828023Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.447858736Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.458535276Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.458566561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.468803818Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.468837335Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.478431313Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.478471495Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.488631094Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.488670217Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.497886595Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.49791712Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.508062152Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.508095031Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.517724097Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.517756413Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.527784588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.527818133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.538102487Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.538143729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.547648481Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.547689948Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.557355288Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.557390183Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.566631815Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.566672419Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.566860823Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.566963707Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660505147Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660555276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660560919Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660573098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660578048Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660581554Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660586298Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660591201Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660596058Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660601027Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660605752Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.66060923Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660612802Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660616336Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660621127Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660625898Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660629368Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660632939Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660636193Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.660651465Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"c266bc0a-8cc5-44e4-8fa8-51cae4a73b3e","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized)]","latency_us":296184,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:47.660857511Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"231.338326ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c266bc0a-8cc5-44e4-8fa8-51cae4a73b3e\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c266bc0a-8cc5-44e4-8fa8-51cae4a73b3e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.894685903Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.894982853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.894992772Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.906054649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.906089047Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.917164447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.917198207Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.928230274Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.928261249Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.93876717Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.938793542Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.950055456Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.950081857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.960161253Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.960189537Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.970789401Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.970817189Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.982329828Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.982359088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.992753829Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:47.992783482Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.00344278Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.003472133Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.013572339Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.013602068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.022368871Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.022397193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.030862059Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.03089086Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.040894756Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.040923079Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.051301093Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.051328524Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.060499393Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.060528784Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.070301798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.070333654Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.079455421Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.079490157Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.089352364Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.08938482Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.099284065Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.09932286Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.099549441Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.099666631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197543568Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197594982Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197601938Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197606785Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197610301Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197615392Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.19761873Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197623492Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197627889Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197631114Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197636936Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197641726Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197645101Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197648251Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.19765172Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197664462Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197668052Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197671184Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197674156Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.197679565Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"fd5cb4e1-945b-494b-b46e-5c02ff895e0b","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized)]","latency_us":302074,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.197830019Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"300.519038ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fd5cb4e1-945b-494b-b46e-5c02ff895e0b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fd5cb4e1-945b-494b-b46e-5c02ff895e0b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.500138933Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.500396216Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.500404729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.511369386Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.511398074Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.521945339Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.521975199Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.531753606Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.53177517Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.541728541Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.541755617Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.552575946Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.552602318Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.561818894Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.561844487Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.571278376Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.57130414Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.579999225Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.580024182Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.589026175Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.589054564Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.597855228Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.59788206Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.606729542Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.606753692Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.617263047Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.61735652Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.626998926Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.627025767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.636986853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.637011219Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.646020584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.646047481Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.654118704Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.654142142Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.662022113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.662048477Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.671717387Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.671746103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.680325621Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.6803528Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.689219169Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.689249283Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.689440659Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.689617332Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779614205Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779655437Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779662272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779667278Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.77967205Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779678462Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.7796833Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779688147Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779691766Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779696257Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779699379Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779704601Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779790764Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779794589Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779798217Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779801715Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779806609Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779810078Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779814699Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.779849266Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5fe3b225-31ca-41b2-9ed3-4561e46c2056","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized)]","latency_us":278827,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:48.779988375Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"158.072856ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5fe3b225-31ca-41b2-9ed3-4561e46c2056\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5fe3b225-31ca-41b2-9ed3-4561e46c2056\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.938852057Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.939141976Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.939148726Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.949695769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.949724307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.960962387Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.960995903Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.972554459Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.972581563Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.981763905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.98179143Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.989996861Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.990028034Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.999959086Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:48.999987521Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.009551572Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.009583522Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.018091354Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.018121421Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.027939613Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.027965966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.038690811Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.03871964Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.047951369Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.047980873Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.05714734Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.05717496Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.06587352Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.065914377Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.075782349Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.075809011Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.085210111Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.085245531Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.094471825Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.094500222Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.102993476Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.10301966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.111270408Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.111297259Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.11939243Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.119419909Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.127293017Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.127325708Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.127529057Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.127641719Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21916637Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219215056Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219223475Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219230825Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21923661Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219242394Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219246416Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219252155Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219266367Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219272039Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21927578Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21927998Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219283617Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219289092Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21929288Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219296826Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21930114Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219306568Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.219312184Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.219316487Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"27c21991-9562-4975-b59a-24e120a330de","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized)]","latency_us":279560,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.21944743Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"398.161097ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27c21991-9562-4975-b59a-24e120a330de\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27c21991-9562-4975-b59a-24e120a330de\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.620143097Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.620570963Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.62058106Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.631572753Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.631599885Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.642786622Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.642810463Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.653223912Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.653249929Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.66334928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.663369708Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.672453941Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.672476489Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.68183227Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.681857001Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.691056838Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.691088423Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.700084436Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.700107021Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.710402777Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.710433978Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.719351826Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.719381771Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.727999229Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.728020521Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.737035247Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.737065472Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.745698848Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.745723083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.753561642Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.753584162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.762604444Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.762628407Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.771394686Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.771488334Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.781199989Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.781228462Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.790007857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.790033733Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.799024157Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.799051927Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.808864375Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.808904098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.809078223Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.809194739Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900606705Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.90065584Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900662205Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900669555Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900675246Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900693397Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900699753Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900707075Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900711088Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900716731Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900720753Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900724822Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900728939Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900734357Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900739907Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900743751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900747942Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900751973Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900757692Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:49.900829122Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"21200918-8cdc-40e3-b129-c6edc4d5558c","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized)]","latency_us":279748,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:49.900974814Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"481.874839ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"21200918-8cdc-40e3-b129-c6edc4d5558c\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"21200918-8cdc-40e3-b129-c6edc4d5558c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.384982873Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.385379841Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.38538646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.397052853Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.397093457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.407918467Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.407946903Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.41896233Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.418986208Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.42880546Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.428834411Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.439049966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.439072184Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.449348656Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.449390103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.46003941Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.460068961Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.470239223Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.470262042Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.480136631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.480161372Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.490315264Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.490343829Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.500688262Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.500715347Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.510780926Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.510804074Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.520081517Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.520107308Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.529064114Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.529084543Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.539472371Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.539560238Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.549235428Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.54926061Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.558814436Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.558842149Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.568081697Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.568108097Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.57802738Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.578052281Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.58806164Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.588104897Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.588289692Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.588393006Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682734124Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.68278775Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682805431Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682810278Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682815996Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.68282224Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682829422Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682833632Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682837569Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682841657Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682847188Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682851266Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682855204Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682859058Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682864758Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682870355Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682874217Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.682955281Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.68295934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.682964853Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"23e9ccb9-72fe-43a1-be5a-58e80ce67fa2","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized)]","latency_us":296906,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:50.683079334Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"292.613772ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"23e9ccb9-72fe-43a1-be5a-58e80ce67fa2\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"23e9ccb9-72fe-43a1-be5a-58e80ce67fa2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.977408669Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.977780399Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.977809208Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.991285275Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:50.99132032Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.002055113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.002083632Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.012740714Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.012768933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.0221051Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.022130245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.031593353Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.031626466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.040974527Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.041003102Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.05118858Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.051214802Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.059745194Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.059770572Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.068696972Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.068723917Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.077736248Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.077772699Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.088305784Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.088333402Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.097401992Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.097433615Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.106356752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.106385862Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.116364848Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.116393012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.12558488Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.125622081Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.13563981Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.135675454Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.144595831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.144623919Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.153550897Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.153577147Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.163678687Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.163712015Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.172745243Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.172787994Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.172985586Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.173139059Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266572363Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266628913Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266633423Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266638477Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266643183Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266648133Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266653168Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266659468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266664239Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266667633Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266670764Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266673999Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266678724Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.26668749Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266691531Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266696143Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.26670085Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266704103Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266707243Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.266710759Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"fd9a97a8-b35f-4e38-8d67-66262574106e","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized)]","latency_us":288384,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.266834803Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"306.101955ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fd9a97a8-b35f-4e38-8d67-66262574106e\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fd9a97a8-b35f-4e38-8d67-66262574106e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.575898354Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.576139944Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.576147978Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.5870157Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.587045663Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.597776997Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.597802094Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.608848687Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.608876765Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.618839986Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.618874665Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.628561904Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.62858162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.639428747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.639455906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.649754416Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.64978017Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.659210418Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.659236679Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.669502405Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.669533109Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.678822698Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.678844993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.688425594Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.688446725Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.697217981Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.69723871Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.706315946Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.706339327Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.716252149Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.716293943Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.726165873Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.726193277Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.735642723Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.73566661Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.744892633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.744920374Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.754535896Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.754567012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.763273986Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.763305347Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.772146834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.772187082Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.772359567Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.772465498Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865772272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865821135Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865830473Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.86583802Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865842173Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865848024Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865852477Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.86585813Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865863802Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865869427Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865887052Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865892771Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865896737Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865900684Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865905025Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865910462Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865916397Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865920559Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.865924544Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:51.865928509Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"858a13f0-1ccd-4ef5-b22a-56206c6e52dd","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized)]","latency_us":289149,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:51.866069035Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"551.657829ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"858a13f0-1ccd-4ef5-b22a-56206c6e52dd\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"858a13f0-1ccd-4ef5-b22a-56206c6e52dd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.421157458Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.4215006Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.421509506Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.433338947Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.433372773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.443886758Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.443915905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.454143816Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.454176198Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.463538255Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.463567694Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.472622512Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.472646213Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.482420634Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.482446154Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.491679384Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.49186788Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.502934524Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.50296803Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.512259725Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.512291103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.521197126Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.52122631Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.530104644Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.530126629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.539577561Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.539600062Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.548803667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.548846503Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.557653072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.557677654Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.568090385Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.568115164Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.577331054Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.577360211Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.586238926Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.586264793Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.595363745Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.595397796Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.60457596Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.604601752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.613285016Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.613315513Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.613501095Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.613617826Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706832865Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706864909Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706871903Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706876844Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706881588Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706885394Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706890063Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.70690241Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706907876Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706914101Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706917542Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706920757Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706923982Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706927113Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706930246Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706934971Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706938074Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706941299Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.706944458Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:52.706949193Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a736b6e2-2944-43df-85f8-5b51ba867317","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized)]","latency_us":284887,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:52.707089952Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"502.512655ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a736b6e2-2944-43df-85f8-5b51ba867317\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a736b6e2-2944-43df-85f8-5b51ba867317\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.211995879Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.212248428Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.212254587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.223710544Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.223746207Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.235173612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.235196928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.245265608Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.245286843Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.254514425Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.25453445Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.263492969Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.263512488Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.273857031Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.273885071Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.283212453Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.283238769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.293331773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.29335568Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.302594406Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.302619427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.31162118Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.311639921Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.321852793Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.321875433Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.331472778Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.331513438Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.340259051Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.340277286Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.349555435Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.34957456Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.358809404Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.358827902Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.367927834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.367947325Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.37682618Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.376850527Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.386477111Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.386499199Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.395939633Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.39596534Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.406162531Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.406194425Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.406499017Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.406611341Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502271012Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502314579Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.50232057Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502339339Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502342949Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502346352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502349802Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502352952Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502357424Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502360715Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502363574Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502366931Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502371712Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502376296Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502379737Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502383009Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502386346Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502389609Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.50239605Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:53.502400943Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ac447f80-7efe-425b-b67b-b15de35f0bea","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized)]","latency_us":289536,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:53.502586107Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.328644947s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ac447f80-7efe-425b-b67b-b15de35f0bea\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ac447f80-7efe-425b-b67b-b15de35f0bea\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.835918807Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.836227967Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.836236732Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.847100218Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.84713935Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.858464933Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.858495158Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.867539972Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.867564553Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.875532409Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.875559476Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.88648457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.886506696Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.897360129Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.897385162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.905927482Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.905951276Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.91599122Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.916013907Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.929188188Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.929215367Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.939816709Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.939839492Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.949006665Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.949032362Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.958763785Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.958785798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.967989221Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.968011028Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.977032973Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.977057226Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.986416224Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.986439896Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.995674485Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:54.995698232Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.004682371Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.004707956Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.014114297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.014140922Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.023070676Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.023099243Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.031170909Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.031194422Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.03138711Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.031500588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123018253Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123054292Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123059017Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123063938Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123067285Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123070735Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123075451Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123078788Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123084742Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123089223Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123093592Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123097332Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123100624Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123104027Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123107327Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123121539Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123125139Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123128458Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123133502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:55.123138171Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"2d11f625-5370-41d1-ac28-5ef7cd33583c","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized)]","latency_us":286303,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:55.123283665Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.059165364s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d11f625-5370-41d1-ac28-5ef7cd33583c\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d11f625-5370-41d1-ac28-5ef7cd33583c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.186606393Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.186986577Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.186993636Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.198394142Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.198420449Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.209534844Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.209560504Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.219481737Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.219509497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.22909416Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.229119127Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.239842729Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.239869823Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.249328734Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.2493564Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.258857967Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.258885156Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.268108985Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.268127876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.276959379Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.276980275Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.286354505Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.286382921Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.295906697Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.295928008Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.304783259Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.304805609Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.313863159Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.313885266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.322612126Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.322634551Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.331589775Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.331618603Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.34043889Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.340461616Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.349627088Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.349653817Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.360053807Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.360076645Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.369266407Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.36935198Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.378132661Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.378168025Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.378454559Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.378596538Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473340178Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473376198Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473384626Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473392332Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473398361Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473404352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473410117Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473415733Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473421742Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473425952Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473431727Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.47343722Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473452006Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473456374Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473460349Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473464287Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473469659Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473473559Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473477478Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:56.473494654Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5c6065ea-689c-4a95-a72d-507ceaf054fd","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized)]","latency_us":285977,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:56.473720848Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"665.588963ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5c6065ea-689c-4a95-a72d-507ceaf054fd\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5c6065ea-689c-4a95-a72d-507ceaf054fd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.142165871Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.142682374Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.142698641Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.153966437Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.154006357Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.165005385Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.165036904Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.175240253Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.175289435Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.184669153Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.184698245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.193761006Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.193800738Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.204411767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.204438749Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.214974162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.215003479Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.223743421Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.22377149Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.232721004Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.232754061Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.241564295Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.241593743Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.25160616Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.251632645Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.261101058Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.261136149Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.270382045Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.270414934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.279389006Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.279423768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.288598187Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.28862847Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.297700916Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.297741266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.306684508Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.306721671Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.31703239Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.317067715Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.326443777Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.326492678Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.336609022Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.336662415Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.336824371Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.336991757Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432407178Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432464639Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432472261Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432478636Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432484833Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432490721Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432495427Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432499214Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432518659Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432524725Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432528887Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.43253514Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432541094Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432546906Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432554716Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.43256051Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432573785Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432578191Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.432584229Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:57.432588832Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6bda9e95-2892-4b14-aa91-e02b9d30d99b","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized)]","latency_us":289401,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:57.43272092Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.199940255s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6bda9e95-2892-4b14-aa91-e02b9d30d99b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6bda9e95-2892-4b14-aa91-e02b9d30d99b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.64056067Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.640836995Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.640844795Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.652852649Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.652905507Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.6638494Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.663895095Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.67486805Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.674906216Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.684048805Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.684094012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.693885065Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.693926579Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.702818707Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.702851914Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.712576224Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.712633975Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.722129768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.722168282Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.731461808Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.731496544Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.740190723Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.740237364Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.749221898Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.749256641Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.757657857Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.757697185Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.765811226Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.765855749Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.774149357Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.774182096Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.783660592Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.783697071Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.793003887Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.793047388Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.801845434Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.801885447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.810677444Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.81072283Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.820161553Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.820208504Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.829107342Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.829148621Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.829342012Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.829442927Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926243278Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926303364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926314914Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926320624Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926324233Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926338483Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926343883Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926349302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926352781Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926367134Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926377737Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926382059Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926386062Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926391725Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926395972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926402554Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926406585Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926410279Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.92641401Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:20:59.926418116Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9328783f-bac6-4d4f-93b2-2864c70a1553","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized)]","latency_us":284944,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:20:59.926565317Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.907476601s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9328783f-bac6-4d4f-93b2-2864c70a1553\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:20:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9328783f-bac6-4d4f-93b2-2864c70a1553\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:01.53013293Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"519.646579ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.840274241Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.840531385Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.840539007Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.852668872Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.852727084Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.864478868Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.864529353Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.875725464Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.875778677Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.885727095Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.885772818Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.895851493Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.89589984Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.906785059Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.906824678Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.917179233Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.917224451Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.92718726Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.927227705Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.938516432Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.938555291Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.948465592Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.948504171Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.95836459Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.958401293Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.968121366Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.96816167Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.977855172Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.977895926Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.987348574Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.987393068Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.997870542Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:01.997912909Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.007317439Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.00735913Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.017424214Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.017464612Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.026724886Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.026778437Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.036259365Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.036296734Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.046894644Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.046941717Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.047147128Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.047266671Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.149848273Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.149907373Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.149914236Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.149921446Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150007079Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150014326Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150019829Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150027227Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150034024Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150040942Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150048302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150053233Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150056946Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150062842Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150068384Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150079559Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.15008359Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150087228Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150090893Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:02.150097428Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9fbaeb89-d1ba-40a3-bdf7-bc08422efbd5","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized)]","latency_us":308866,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:02.150261185Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.634876197s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9fbaeb89-d1ba-40a3-bdf7-bc08422efbd5\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9fbaeb89-d1ba-40a3-bdf7-bc08422efbd5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.793549601Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.793815332Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.793824613Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.805217709Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.805270904Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.816602747Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.816642479Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.827381887Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.827424094Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.839353299Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.839398675Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.849959302Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.850002278Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.861179181Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.861213954Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.871095146Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.871141291Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.881502584Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.881541398Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.892246897Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.892288503Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.902829325Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.90287294Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.912872722Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.912912754Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.923034998Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.923067577Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.932909746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.932956284Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.942867823Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.942906271Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.952902379Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.952943986Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.961627876Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.961673214Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.972256925Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.972302039Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.982959736Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.983010297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.993347064Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:04.995735041Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:05.003604442Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:05.003644193Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:05.003914633Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:05.004044704Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100783466Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100848343Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100857961Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100865443Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100872277Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.10087779Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100881634Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100886903Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100890803Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100897Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100902569Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100921994Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.100992502Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101003302Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101007183Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101017177Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101022586Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101028361Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101031615Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:05.101035477Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"758b4da7-3a35-4700-92ff-bc31ababd20b","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized)]","latency_us":306504,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:05.101187871Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.570910133s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"758b4da7-3a35-4700-92ff-bc31ababd20b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"758b4da7-3a35-4700-92ff-bc31ababd20b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.679674406Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.680092056Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.680102838Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.692432446Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.692475601Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.704274487Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.704307244Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.714985468Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.71508457Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.726641012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.726678175Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.736454756Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.736488179Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.745613833Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.745644012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.755384155Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.755417972Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.765314959Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.765344941Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.775025239Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.775054687Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.784970277Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.7850025Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.79458659Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.794616768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.804543384Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.804569532Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.814841459Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.814873588Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.824176753Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.82420937Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.834039701Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.834069692Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.844206735Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.844243824Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.854221069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.854252551Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.864077113Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.864110985Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.873723101Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.873757658Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.883244807Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.88328197Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.88352445Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.883630831Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983702624Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983822276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983829973Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983836939Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983842583Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983854189Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983870474Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983875818Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983887462Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983892975Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983896819Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983902106Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983909081Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.98391435Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983918148Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983922029Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983926201Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983931169Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.983941191Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:07.983955835Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"1c8428ed-4634-4f23-b151-8f96751ae432","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized)]","latency_us":303251,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:07.98413494Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"3.621917552s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1c8428ed-4634-4f23-b151-8f96751ae432\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1c8428ed-4634-4f23-b151-8f96751ae432\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.616466069Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.616813929Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.616821447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.628257447Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.628313923Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.640514369Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.640571042Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.651217152Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.651261188Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.661439859Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.661490766Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.672520173Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.672556905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.682209253Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.682251938Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.692467834Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.692519942Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.701953376Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.702004977Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.712133103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.712199179Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.721373762Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.721409706Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.732483602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.732528237Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.74250332Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.742557791Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.751920302Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.752034305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.762482456Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.762519441Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.771785232Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.771830899Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.780914237Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.780957916Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.7914108Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.791456832Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.801851928Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.801896019Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.811181977Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.811224484Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.821046016Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.821096454Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.821352383Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.821474843Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.923974243Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924030972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924045597Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924052825Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.92407377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924080351Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924084223Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924089098Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924092811Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924096374Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924101399Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924106461Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924111589Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924116511Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924120477Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924124424Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924127874Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924132636Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924136208Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:11.924167784Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"dfdfadf8-4b19-43a8-83da-a9bd80f276bf","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized)]","latency_us":306729,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:11.924336238Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"5.628050737s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dfdfadf8-4b19-43a8-83da-a9bd80f276bf\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dfdfadf8-4b19-43a8-83da-a9bd80f276bf\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:16.023565373Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39588"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:16.023652515Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1bfb6d8a-b3af-4c9d-8988-acf71d3eed1a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39588","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":25,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.186590858Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"1.88973069s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.568980973Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.569234685Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.569243272Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.580816841Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.580871198Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.591312478Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.591354722Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.600956342Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.600995474Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.610603333Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.610658515Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.621759812Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.6218083Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.631540804Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.631587249Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.640602465Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.64064996Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.649036398Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.649082439Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.657332966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.657372258Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.666539291Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.666590761Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.675492929Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.675532717Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.684412222Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.68445773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.693194418Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.693239803Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.702041959Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.70214411Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.711542966Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.711587479Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.720046138Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.720157962Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.728659041Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.72869702Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.737668524Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.7377103Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.747071996Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.747116762Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.755753968Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.755799091Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.75597484Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.756080245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851855283Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.85191521Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851922972Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851930313Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851937747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851952795Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851959751Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851964064Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851969483Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851973364Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851978915Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.851996943Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852001209Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852006468Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852010359Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852015759Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852019565Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852030216Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852037313Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:17.85204141Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ae3d1eab-41d1-457f-a70c-225a6663299b","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized)]","latency_us":282197,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:17.852208682Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"4.612188533s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ae3d1eab-41d1-457f-a70c-225a6663299b\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ae3d1eab-41d1-457f-a70c-225a6663299b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.478271986Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.478524151Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.478532026Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.489642427Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.489699297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.501243823Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.501286368Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.51087079Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.51092452Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.521430486Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.521477934Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.532272355Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.532311205Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.54232059Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.542364166Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.551788131Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.551836798Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.562699565Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.562749769Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.573823373Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.573870906Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.583901861Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.583950012Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.595379035Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.595417858Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.604539191Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.604578379Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.613181619Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.613236689Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.62388305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.623922069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.634833772Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.634878238Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.643972268Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.64402221Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.654450219Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.65456217Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.663589469Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.663632548Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.674003375Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.674048826Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.684643662Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.684699772Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.684879674Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.684998035Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785620942Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785679847Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785686306Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785692053Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785700763Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785706519Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785712294Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785718885Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785729057Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785734457Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785739651Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785743907Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785747507Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785755785Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785762826Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785768002Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785771564Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785775446Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.78577928Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:22.78578303Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"0bce6918-5bd6-4ca9-be58-5328f03e1df0","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized)]","latency_us":306596,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:22.785989732Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"5.468438531s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0bce6918-5bd6-4ca9-be58-5328f03e1df0\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0bce6918-5bd6-4ca9-be58-5328f03e1df0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.271232674Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.271503459Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.271510997Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.283530993Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.283586336Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.295120732Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.295179289Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.305297306Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.305338754Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.316129569Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.316177148Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.325723773Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.325763037Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.335549801Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.335589768Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.344769063Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.344820917Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.353188128Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.353224746Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.362692006Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.362729667Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.373731251Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.373772277Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.383215266Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.383258842Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.391595139Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.391646224Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.39981611Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.399858758Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.407838521Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.407884787Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.418177628Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.418215204Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.427258563Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.42730213Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.43546069Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.43550073Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.443400466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.443440461Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.451280824Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.451328478Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.46095262Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.460994452Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.461269269Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.461453595Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562595111Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562671979Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562679732Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562685207Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562705627Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562711092Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562716408Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562731371Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562737277Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562740915Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562746081Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562749924Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562753787Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562757509Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562763659Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562770231Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562774056Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562777863Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562781425Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:28.562785316Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a12bca9e-6a2d-489b-aac2-885e297a1cbb","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\", TimeNotSynchronized), (\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\", TimeNotSynchronized), (\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\", TimeNotSynchronized), (\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\", TimeNotSynchronized), (\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\", TimeNotSynchronized), (\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\", TimeNotSynchronized), (\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\", TimeNotSynchronized), (\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\", TimeNotSynchronized), (\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\", TimeNotSynchronized), (\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\", TimeNotSynchronized), (\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\", TimeNotSynchronized), (\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\", TimeNotSynchronized), (\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\", TimeNotSynchronized), (\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\", TimeNotSynchronized), (\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\", TimeNotSynchronized), (\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\", TimeNotSynchronized), (\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\", TimeNotSynchronized), (\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\", TimeNotSynchronized), (\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\", TimeNotSynchronized)]","latency_us":290599,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:28.562920103Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"4.605693396s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a12bca9e-6a2d-489b-aac2-885e297a1cbb\", \"content-length\": \"124\", \"date\": \"Fri, 20 Dec 2024 21:21:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a12bca9e-6a2d-489b-aac2-885e297a1cbb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: f3f6b067-d10e-44b3-9f21-f8e5a96a2b37 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4565f09e-ee63-469d-bb6e-3ccd4abd01fe (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73abfaca-4f35-4f97-9980-1890d6dba8fc (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: a202ea32-7b9e-4e81-a217-057c37186ede, kind: Service { id: b6c9cbef-5cb0-4145-b5d4-2d3d002e7390 }, name: Name(\"ntp-b6c9cbef-5cb0-4145-b5d4-2d3d002e7390\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 228, 3])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: dbf50d9a-906f-459e-a09e-f219a42e264d (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: 50bfb30f-8e0c-4b57-a131-782097c61910 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4728b7fd-e948-4a22-a906-8df24c162980 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: ad844bf5-de77-449c-b06f-50b2df3f4d97 (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4939e8c6-7a63-45f9-a24a-89f71cba5b00 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f413b9dc-12d9-42dc-9e49-cb25b63f7e80 (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 1546fa03-3745-40e3-a644-64bff34101ed (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: c824ab3d-7cad-4d48-adf0-dd30df7034a3 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: db3b99c3-ae90-44ff-b332-4240930102b5 (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 919e46ed-7d75-489e-b206-ab319ccaf2ad (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 35b4cc1c-ed13-46ec-8b86-d499bd7e1694 (service), filesystem_pool: Some(ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External } } } }, OmicronZoneConfig { id: b37ee4ad-9b00-4df2-a7ec-46bfe7940061 (service), filesystem_pool: Some(ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99fac52c-805b-41c4-9b07-93a7feb7a322 (service), filesystem_pool: Some(ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External } } } }, OmicronZoneConfig { id: 81fe9867-92d4-4989-a410-474fee97f4bb (service), filesystem_pool: Some(ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External } } } }, OmicronZoneConfig { id: d7837344-d1e9-4005-8708-29709de8b650 (service), filesystem_pool: Some(ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External } } } }, OmicronZoneConfig { id: ea56f3ce-62af-4852-a856-bc1cf6555fc2 (service), filesystem_pool: Some(ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External } } } }, OmicronZoneConfig { id: d4fff15d-4a10-48f7-964f-a50935e07aee (service), filesystem_pool: Some(ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External } } } }, OmicronZoneConfig { id: c49f3484-1f36-4740-8526-55691d8a27dc (service), filesystem_pool: Some(ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8060c010-a1c1-48e7-8bbe-b43703a8b5db (service), filesystem_pool: Some(ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.181762251Z","hostname":"buskin","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.182102643Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.182110501Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.194429621Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.194489753Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.205150564Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.205192115Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.215220317Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.215259721Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.226749101Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.226790774Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.237573193Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.237617323Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.248988687Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.249039782Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.259986952Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.260029581Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.27082787Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.270868164Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.281419905Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.281464952Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.292709956Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.292753776Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.305443344Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.30549307Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.316745602Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.316793846Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.327032713Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.327074545Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.337654298Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.337713053Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.347017587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.347055528Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.357670305Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.357721072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.367437838Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.367545297Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.378076155Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.378117163Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.387751797Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.387787442Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.397532874Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.397648197Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.397819013Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.397933953Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.520537553Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.52130518Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.521327897Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.524089416Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.524183121Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.563485577Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.563661587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.563698788Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.563705144Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.577922846Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.74436156Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.744604625Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.744616671Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.744620581Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.758251689Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.810947792Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.810991687Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.811004587Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.811008506Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.811105015Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.81631166Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.822418278Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.82921971Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.835425157Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.84094803Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.845715341Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.993815387Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:33.999813595Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.005694337Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.011539721Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.017291108Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.022723299Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.028033301Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.033102148Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.038949Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.044631463Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.050006707Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.055086263Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.062070293Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.070295301Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.141063463Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.14164868Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.15273207Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.163732514Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:34.17221527Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"3.259843864s"} {"msg":"Installing Omicron zone: oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.173361892Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.189974892Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.197643736Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.204614257Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.272437439Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.43207276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.496674226Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.556528257Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.559313777Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.559343713Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.614461261Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 164, 161]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.61449549Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.618006382Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.618027983Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.676293359Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 163, 196]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.676322779Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.678471311Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.678487276Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.738946034Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 178, 2]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.738978697Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.739263417Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.739278689Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.740731566Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.740743426Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.798752259Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 187, 173]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.798781369Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.80107383Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.801086637Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.858225845Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 237, 64]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.858255424Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.858615044Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.860012072Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.942903609Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.942984882Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.943209756Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.943228394Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.943611034Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.943631115Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.943720396Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.943801679Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.94385074Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.948242422Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.953045894Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.953068236Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.953082767Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.956516993Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.960050921Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:34.960069793Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.084859299Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.19316646Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.202896806Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.210572901Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.219372163Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.227718465Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.236167442Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.243504231Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.251218322Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:35.260169308Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.378597129Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.398735612Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.39882504Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.536273972Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.665046207Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.738885439Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"0f96c78f-a449-48f4-8beb-6a7854377da5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65307","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31847,"response_code":"200"} {"msg":"Profile for oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.784190522Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.822815766Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.879916032Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.921037222Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.986674295Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:46.989324254Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.061099855Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.06206335Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.062402282Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.091414927Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.184996058Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.199571481Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.207250305Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.224192034Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.270628359Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.271226534Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.399405275Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.399710428Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.635638128Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.636155914Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 73.212014ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.82126686Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 40.855294ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.833370436Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 47.007111ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.845532225Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 45.25477ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.858250042Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 44.959127ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.870805349Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 43.539192ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.882699452Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 28.528823ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.893299145Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 52.189641ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.905054783Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 73.568401ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.96851846Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 42.343335ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.983999991Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 66.341273ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:47.996924068Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:47.997473954Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 93.729394ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.013185113Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 46.180721ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.025879111Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 143.833964ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.048325026Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 78.550737ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.097536847Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 68.780119ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.110906273Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 84.93092ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.123391962Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 102.771803ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.135829988Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 102.178973ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.147815436Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 246.083438ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.159876447Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 149.69048ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.172099161Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 95.398928ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.185458064Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:48.185524066Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 31.984409ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.253292163Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 278.250511ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.269071889Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 144.938747ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.283471047Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 166.941891ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.298186756Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 215.833129ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.312096459Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 268.300991ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.329380469Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 267.588128ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.342554065Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 194.487971ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.354715499Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 279.449702ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.366540307Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:48.36771486Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 116.888588ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.382608486Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 281.271525ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.393524782Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:48.394660203Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 146.574736ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.408640367Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:48.408791439Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 428.386737ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.425497127Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5 zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:48.425627452Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 158.501276ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.445615033Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 53.547288ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.474826543Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 215.292498ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.484872007Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 234.624242ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.512788624Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 579.235062ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.5420568Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 134.790263ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.551433053Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 560.16051ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.562418146Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 328.656065ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.572371381Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 108.454737ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.584004795Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 303.740799ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.611919101Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 254.136886ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.624350727Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 356.118521ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.639684886Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 37.369499ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.651707884Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 481.152161ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.661188256Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 532.144963ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.735016649Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.044517491s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.745812104Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 77.089503ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.755253074Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 162.166539ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.764501161Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 36.189202ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.775134257Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 328.447968ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.787960929Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 876.742759ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.799130641Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 83.337321ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.821525094Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 155.078167ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.845928399Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 497.913939ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.868631625Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 472.105603ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.893010264Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.046652056s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.916674199Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 165.345954ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.929345141Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.041320781s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.942401293Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 221.625493ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:48.956219521Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.189323323s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.035182742Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 266.141603ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.049623527Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 329.560201ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.112444539Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 868.927532ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.13005939Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.159579162s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.145684073Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 565.628201ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.158375751Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 710.081193ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.173172514Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 1.043515456s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.190681777Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 1.002695251s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.282372835Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 890.566731ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.330454521Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.191026469s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.378529964Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 1.203383501s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.390793877Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.097043309s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.458733687Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 1.007680549s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.69308957Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 1.204332855s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.742979639Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.045540535s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.811775398Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 824.290285ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.900381224Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.044026487s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.981944233Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.174741194s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:49.998297201Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 632.484484ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.012790316Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 501.753408ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.252617043Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 617.607058ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.270883583Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.069412436s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.280925538Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 518.977624ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.298520533Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.427030434s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.346750664Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.468131965s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.567164537Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 783.815128ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.581443717Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 1.027184039s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.604709569Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 724.105747ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.65605703Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:50.684641961Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:50.685096731Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:50.691793861Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:50.692314873Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:50.698389769Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:50.698514932Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 1.496074261s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.71356647Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 1.416070799s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.734922178Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 633.264738ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.764569033Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 982.028464ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.829169842Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 52.49071ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.86344986Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 862.967654ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.872145895Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 32.347918ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.879183503Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 66.454339ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.886310403Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 745.749716ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.897963074Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 133.27297ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.919135119Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 71.173079ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.925917671Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 677.040761ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.956695471Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 149.132805ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:50.964955136Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 115.693983ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.006572869Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 952.59681ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.036715335Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 277.640637ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.063625208Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 104.326867ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.139222284Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 522.556876ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.155302124Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 620.692014ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.184324695Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 458.760787ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.260905862Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 518.348788ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.351638667Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.139289675s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.360928065Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.026158895s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.37496596Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 568.285051ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.389712371Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 1.379260692s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.406957817Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 561.526941ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.642123231Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 1.098536679s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.651322727Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 943.12844ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.661150267Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 1.188063641s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.687487468Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 998.281642ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.729814447Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.030890636s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.744624909Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 971.495579ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.792979303Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 962.802882ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.818595368Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 524.743393ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.830108326Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 979.558039ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.908940614Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 675.352404ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:51.971309302Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.322982544s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.009445437Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.197158516s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.047994052Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 614.086543ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.162340536Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 996.497593ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.214672259Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 1.332017703s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.224855988Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 1.076164947s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.365757287Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.277825473s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.414245614Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 692.664985ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.513580792Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 1.239074798s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.616426278Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 808.38477ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.658519302Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 529.910815ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.739783794Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 1.015771095s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.761261266Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 864.37285ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.776455987Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.374137168s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.786289187Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.479543185s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.795994515Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 959.478679ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.804464205Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 673.058299ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.813022496Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 1.475158908s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.885694145Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 818.685607ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:52.898312878Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 710.967536ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.218396336Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 965.560008ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.228005961Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 942.917336ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.255475877Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 1.14082115s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.280707353Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.268129499s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.343645204Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 558.451646ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.452054205Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 1.47085695s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.476573185Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 1.3086603s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.497243979Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 823.608408ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.566912311Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.410242359s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.65117325Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.131339335s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.703547085Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 1.052451838s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.728516675Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 611.516444ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.774276931Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 895.04307ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.787471907Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 1.157539162s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.866898048Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 709.832915ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:53.942553174Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 774.646812ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.026512757Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 662.430588ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.175535269Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 723.075232ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.208288183Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 816.539876ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.222102733Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.37648701s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.289412432Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.339870594Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"5.179796797s"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 1.228435363s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.376138187Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 1.228044211s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.399968682Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 1.407299817s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.412326339Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 746.368585ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.434736821Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 750.972564ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.625276834Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 930.904887ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.668630336Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 1.247364985s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.697999685Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 879.120577ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.798896338Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 1.017869333s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.816912222Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 532.77444ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.828717046Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.293701623s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.845604655Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 572.190993ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.855154219Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 837.234793ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.944620465Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 840.49021ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:54.961043813Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 782.720353ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.038147002Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 904.870608ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.051086842Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.309687544s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.080269727Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 1.202918251s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.196564404Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 849.87992ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.500699945Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.480743464s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.518141256Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 867.539003ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.530900262Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.302326203s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.613584516Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 916.150557ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.624930041Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 827.805657ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.639821895Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 906.006347ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.675085955Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 1.372231864s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.691188437Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:55.720534015Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:21:55.720707239Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 1.286754101s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.793599303Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 936.800884ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.813687304Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 590.292537ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.830857789Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 814.701908ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.840448674Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 706.551821ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.849797984Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 753.972973ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.961482547Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.425379335s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:55.977867735Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 49.043764ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.040745325Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 90.682152ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.099091523Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.306429497s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.148870279Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 274.313755ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.201104441Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 707.923695ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.363910252Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 608.411349ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.403614402Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 949.181082ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.41420102Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.373604513s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.424695105Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 687.408344ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.435345414Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 1.082955703s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.477704564Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 204.923854ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.489307224Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 1.20315947s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.564343607Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 955.776799ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.575483666Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.290454145s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.59572829Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 963.783915ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.668811129Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 1.067566831s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.707907666Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 649.766396ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.740970877Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 1.389237847s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.76336909Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 958.198801ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:56.931345876Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 716.250167ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.011980794Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.117053753s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.023360189Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 631.040682ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.076406529Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 1.084004866s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.087214201Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 758.154071ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.098176727Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 748.818044ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.134692536Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 1.100043835s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.378864632Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 621.439513ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.407192777Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.219491737s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.421762143Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.137461477s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.469135646Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 859.404105ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.545721003Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 521.825633ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.573936076Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 839.31426ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.665724434Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 953.135986ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.731380207Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 644.722173ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.757489582Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 830.772171ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.789436528Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 608.892292ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.812908775Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.272813257s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.832803161Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 1.150898335s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.879300583Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 1.434780657s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.904206632Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.450878968s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.920566279Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.447815182s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:57.943714758Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 750.506719ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.060697531Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 800.171148ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.121334352Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.051938217s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.172614822Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 989.915582ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.207968725Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 1.246461899s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.241862366Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 573.479481ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.429462184Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 1.245253483s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.462158578Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 1.233462891s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.49259114Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 1.254580464s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.525954279Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 1.252360748s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.548097976Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 1.380431041s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.628019458Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 885.812459ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.647068436Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.465068987s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.666273237Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 689.345269ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.699622805Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 853.980747ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.829206292Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 608.694399ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:58.939538926Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.076538316s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.022450726Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 1.094710791s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.048598803Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 1.049685484s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.119553735Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 1.435556007s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.215735822Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.121655537s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.236406156Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 909.39229ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.35781924Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.443262615s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.389593173Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 1.347988245s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.404596659Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.450930235s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.42024416Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 880.95265ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.504759966Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 708.799371ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.600188267Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 1.122645207s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.613329432Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 680.392684ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.698609305Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 1.214163525s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.725302573Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 1.427995163s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.740614077Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 1.133448834s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.793913783Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 972.902238ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:21:59.812869768Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 672.30311ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.023137244Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.385538792s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.119045206Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 969.551289ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.147507354Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 797.502529ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.164965896Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 655.499131ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.188263209Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 824.387043ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.282696837Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 756.504231ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.321454206Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.402832122s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.368994313Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 1.427052006s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.38934627Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 1.332442658s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.398966105Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 612.476824ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.663972361Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 527.147116ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.705628945Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 538.43661ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.746497672Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 647.041662ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.761904978Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 612.810962ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.793957317Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3\") failed: Property not found. retry in 1.05520379s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.841155885Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 661.057426ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.852673374Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 685.866972ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.881568871Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 595.024166ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.936560315Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 1.115741402s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.947647884Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5\") failed: Property not found. retry in 1.163150183s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:00.971291804Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 511.061309ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.088937003Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 962.984837ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.116986382Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5\") failed: Property not found. retry in 1.429539835s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.126982907Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 784.099226ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.178216706Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad\") failed: Property not found. retry in 891.090758ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.242289903Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8\") failed: Property not found. retry in 1.254299601s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.285277078Z","hostname":"buskin","pid":1566,"zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb\") failed: Property not found. retry in 869.428429ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.295245811Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80\") failed: Property not found. retry in 1.274286755s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.416167844Z","hostname":"buskin","pid":1566,"zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\") failed: Property not found. retry in 829.516985ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.42670593Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee\") failed: Property not found. retry in 1.466458994s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.514563521Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 587.618708ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.523719317Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed\") failed: Property not found. retry in 751.867254ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.541737421Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2\") failed: Property not found. retry in 1.141685658s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.580878869Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f\") failed: Property not found. retry in 668.652852ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.609737817Z","hostname":"buskin","pid":1566,"zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694\") failed: Property not found. retry in 892.304568ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.74539994Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc\") failed: Property not found. retry in 1.114676334s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.783935544Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db\") failed: Property not found. retry in 1.02092942s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.826486439Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 506.578943ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:01.973015436Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650\") failed: Property not found. retry in 633.916163ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:02.078762278Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322\") failed: Property not found. retry in 1.448725081s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:02.095026202Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061\") failed: Property not found. retry in 800.990723ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:02.129392283Z","hostname":"buskin","pid":1566,"zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 1.18331985s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:02.500794132Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 1.032580294s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:03.705099238Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 691.264622ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:04.759530793Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 583.804568ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:05.467088089Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:05.941746303Z","hostname":"buskin","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"15.06039103s"} {"msg":"wait for service Some(\"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed\") failed: Property not found. retry in 761.421791ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:22:06.068505655Z","hostname":"buskin","pid":1566,"zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861449107Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_d4fff15d-4a10-48f7-964f-a50935e07aee"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861512828Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_c49f3484-1f36-4740-8526-55691d8a27dc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861524379Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_0a5aaad3-b9af-4668-92b2-6758e325a2ed"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861535219Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_c824ab3d-7cad-4d48-adf0-dd30df7034a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861540919Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_12bea37a-3ccb-4c9f-8c92-9d6095867a0f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861545879Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_919e46ed-7d75-489e-b206-ab319ccaf2ad"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861565879Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_8060c010-a1c1-48e7-8bbe-b43703a8b5db"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.86157154Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_81fe9867-92d4-4989-a410-474fee97f4bb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.86157683Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_external_dns_065cea45-0eca-4287-90a0-2c3c20ec32f8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.8615819Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_db3b99c3-ae90-44ff-b332-4240930102b5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.86158677Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_99fac52c-805b-41c4-9b07-93a7feb7a322"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.86159158Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_d7837344-d1e9-4005-8708-29709de8b650"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.8615964Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_oximeter_f413b9dc-12d9-42dc-9e49-cb25b63f7e80"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.86160115Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861605711Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_external_dns_b24cb173-c3a3-4237-abe6-0e7953a415e5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861613241Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_clickhouse_1546fa03-3745-40e3-a644-64bff34101ed"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861618761Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_35b4cc1c-ed13-46ec-8b86-d499bd7e1694"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861623911Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_b37ee4ad-9b00-4df2-a7ec-46bfe7940061"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.861628431Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_ea56f3ce-62af-4852-a856-bc1cf6555fc2"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.862832538Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.869692691Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.88076492Z","hostname":"buskin","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"9f1a0fc6-ab13-4f2f-9763-7efaabea89b8","remote_addr":"[fd00:1122:3344:101::1]:52963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33655540,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.894914447Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1415"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.911695583Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:790"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.914136058Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:802"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: Some(\n None,\n ),\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.946576215Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1016"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:06.950395601Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:07.025478094Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:07.125877176Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:07.306392404Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:07.468762965Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:07.662208053Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:07.864707294Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:08.245320618Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:08.683955255Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:09.05344495Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/a7e68df9-95f5-4fda-aae0-2f46483e29ac/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:09.529952826Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dfa800ce-f140-48f6-95b7-bd49c2669f3d\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:13 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"a7e68df9-95f5-4fda-aae0-2f46483e29ac\\\"\", request_id: \"dfa800ce-f140-48f6-95b7-bd49c2669f3d\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:13.049484499Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75933f4f-7b57-429f-85e0-89aace5ae708\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:13 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"a7e68df9-95f5-4fda-aae0-2f46483e29ac\\\"\", request_id: \"75933f4f-7b57-429f-85e0-89aace5ae708\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:13.865937908Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"915616c6-c47c-46ed-a26f-9ea57969e557\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:16 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"a7e68df9-95f5-4fda-aae0-2f46483e29ac\\\"\", request_id: \"915616c6-c47c-46ed-a26f-9ea57969e557\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:16.374792995Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:16.790203571Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35321"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:16.790279082Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1e33b30c-bf60-4ea7-b805-506071de63ff","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35321","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"460b152c-27ef-425a-81e9-4eac8d2b1cf2\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:17 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"a7e68df9-95f5-4fda-aae0-2f46483e29ac\\\"\", request_id: \"460b152c-27ef-425a-81e9-4eac8d2b1cf2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:17.254313567Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7882067e-6cd1-4f89-a563-13cf48e117ff\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:18 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"a7e68df9-95f5-4fda-aae0-2f46483e29ac\\\"\", request_id: \"7882067e-6cd1-4f89-a563-13cf48e117ff\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:18.950825229Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c55f06f-b130-45c3-8f95-decf865006d4\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:22 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"0c55f06f-b130-45c3-8f95-decf865006d4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:22.233318473Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1b6a1f2-4f21-4db4-97ed-c8a5a85cc1a1\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:24 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"b1b6a1f2-4f21-4db4-97ed-c8a5a85cc1a1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:24.930434131Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8e15cf7-fa63-4253-9d9a-109122b8ade1\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:26 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"b8e15cf7-fa63-4253-9d9a-109122b8ade1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:26.491617208Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00f0fa9a-58d8-4f78-bfa5-a8d605692fe8\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:30 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"00f0fa9a-58d8-4f78-bfa5-a8d605692fe8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:30.309237501Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ff7ed7c4-038a-42c7-8a67-5b1a18738f08\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:32 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"ff7ed7c4-038a-42c7-8a67-5b1a18738f08\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:33.184231907Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"62257ffc-877e-4a9b-a81f-19d4a65b2630\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:36 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"62257ffc-877e-4a9b-a81f-19d4a65b2630\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:36.874950784Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b1099a9e-fa4d-46c8-bed6-aafbe5954565\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:41 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"b1099a9e-fa4d-46c8-bed6-aafbe5954565\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:41.315443422Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:46.824955412Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41933"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:46.825051565Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"10d3b73f-0040-49f8-af4a-52df904c223a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41933","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c3d498cb-c211-4912-8a46-fb457d733788\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:47 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"c3d498cb-c211-4912-8a46-fb457d733788\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:48.447020663Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0e335a74-7377-407f-b4c1-7bd33cd47318\", \"content-length\": \"173\", \"date\": \"Fri, 20 Dec 2024 21:22:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8c74ff24-982c-40fd-86fd-ef04898c5922\\\"\", request_id: \"0e335a74-7377-407f-b4c1-7bd33cd47318\" }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:56.117226282Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1078"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:56.541360173Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:54456"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:22:56.581934633Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"fb7d37de-cf9d-46c2-981e-e5f899744669","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39616,"response_code":"200"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.664400714Z","hostname":"buskin","pid":1566,"component":"NexusNotifierTask","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.750326551Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36577"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.752756105Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.752895548Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.753212825Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.754512865Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.76634532Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.766400151Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.791712779Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.791748999Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.814735685Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.814774476Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.834538099Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.83458829Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.855645402Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.855689083Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.877297497Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.877345258Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.896712832Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.896751623Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.916376583Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.916425374Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.937525397Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.937575519Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:00.948946174Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"b8cc6021-4724-4301-8d37-f1ed9fcd5735","remote_addr":"[fd00:1122:3344:101::b]:36577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198257,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.731466864Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49291"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.739832961Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740279671Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740313542Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740323922Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740329702Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740361833Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740371783Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.740400894Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d9e8bbea-300b-428c-a752-1b9be35b7aa0","remote_addr":"[fd00:1122:3344:101::a]:49291","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8333,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.741232072Z","hostname":"buskin","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1088"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.744515986Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:05.745366355Z","hostname":"buskin","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.104293423Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40018"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.104431356Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.104438976Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.104471727Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.104792204Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.117361926Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.117407197Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.143927282Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.143968293Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.167423199Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.16747786Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.188959682Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.189027003Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.212180452Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.212232133Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.232685162Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.232746593Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.253205372Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.253256473Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.272626488Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.272663008Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.293969766Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.294008707Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.307351946Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"6b821505-87bc-4e07-a779-1bd6e896dd1a","remote_addr":"[fd00:1122:3344:101::a]:40018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202943,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.486958714Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53126"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.48721867Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487285071Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487328412Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487344453Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487358683Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487379103Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487401384Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.487423894Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7591d766-6d51-488b-a843-4355d141ff69","remote_addr":"[fd00:1122:3344:101::a]:53126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":381,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.891089515Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42066"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892119358Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892169749Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892202Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.89221994Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892226151Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892231511Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892246831Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:06.892251961Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8e25908d-7aae-4fa6-82ab-492a7a92a8fb","remote_addr":"[fd00:1122:3344:101::b]:42066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.67797464Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53095"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678160904Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678203345Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678234906Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678242906Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678248906Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678259636Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678265417Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:07.678275697Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"13ae0525-c427-4710-977c-151094e743bf","remote_addr":"[fd00:1122:3344:101::c]:53095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":241,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:10.387180099Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57570"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:10.387461155Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"56058326-9e00-47d1-aecf-b8720e19864c","remote_addr":"[fd00:1122:3344:101::a]:57570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":215,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:10.418697655Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:10.41891453Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d62595cb-880b-4d0f-8747-29290eab7398","remote_addr":"[fd00:1122:3344:101::a]:57570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1759,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:10.523817593Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"246a015c-3f9c-45a9-bdad-4dcc583374b9","remote_addr":"[fd00:1122:3344:101::a]:57570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3610,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:16.858691961Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:16.858791433Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"43bd1ee9-d0ba-4e8f-9b01-21b341e29ca2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59877","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:19.800669989Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59712"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:19.800848993Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a17fe5fb-a3ec-41f9-a231-5a14e04f1a98","remote_addr":"[fd00:1122:3344:101::b]:59712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:19.850292132Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:19.850336203Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4c28aa26-d4fc-4d7e-b86a-d13bdcfbcf6d","remote_addr":"[fd00:1122:3344:101::b]:59712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:19.860589693Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f0f1e625-e36e-4c95-9879-013a3f40b612","remote_addr":"[fd00:1122:3344:101::b]:59712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.145353017Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63043"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.145859719Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"98702811-1043-46ac-83b0-09e2f168999e","remote_addr":"[fd00:1122:3344:101::b]:63043","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":463,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.226133839Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33873"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.2319798Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.344045663Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.410474142Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d5c35e8f-39f9-4ca1-8493-5da50af41422","remote_addr":"[fd00:1122:3344:101::b]:33873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184240,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.410655006Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56463"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.410720498Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40148"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.410935523Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"08cfc9b7-ea39-48f9-9096-64dd35bc5019","remote_addr":"[fd00:1122:3344:101::a]:40148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.411053306Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"534ca97c-e763-4718-b778-78662e152a9c","remote_addr":"[fd00:1122:3344:101::a]:56463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.41481221Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c0f7e62d-2fb8-4d09-8499-a6b910d50472","remote_addr":"[fd00:1122:3344:101::b]:33873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4074,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.427524035Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53454"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.427691208Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ffcb47f1-45a6-44a1-a216-8624555aa6ac","remote_addr":"[fd00:1122:3344:101::c]:53454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.494977147Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57798"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.495319155Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.544794814Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.606937238Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"28b590da-8ef8-4826-95eb-0a169dd026ff","remote_addr":"[fd00:1122:3344:101::a]:57798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111835,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.607574562Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.607598103Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.607607533Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"821e8055-d6b4-48d8-b527-aec8ac75ac02","remote_addr":"[fd00:1122:3344:101::c]:53454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":343,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.607634303Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5749bf68-679e-4806-b7a0-16a5b89a8e9c","remote_addr":"[fd00:1122:3344:101::a]:40148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":469,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.607684655Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c1b6ec0d-e8ba-46fd-a219-d80cb3bf64a4","remote_addr":"[fd00:1122:3344:101::a]:57798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":413,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.609315461Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.609459524Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.612020152Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.612700237Z","hostname":"buskin","pid":1566,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.612726118Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"PUT","req_id":"96f7e3e9-b1a0-48d0-93b0-1128f7d3a79c","remote_addr":"[fd00:1122:3344:101::b]:33873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5489,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.618779253Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"20304f2e-e7d1-48f1-8ca6-b0b3df80b898","remote_addr":"[fd00:1122:3344:101::a]:40148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.620289827Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6da7959a-f922-48af-955c-f0e864754c68","remote_addr":"[fd00:1122:3344:101::c]:53454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.715431071Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53581"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.715540903Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.752960412Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.807257Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b4cd1a84-70f7-4583-a571-6505bd4909cf","remote_addr":"[fd00:1122:3344:101::c]:53581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91769,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.807367512Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52743"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.807526756Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2e9e695c-b038-4404-8dc0-fca26a992050","remote_addr":"[fd00:1122:3344:101::c]:52743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:20.807657889Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"15ed1b17-abf2-46bd-8a93-1aba39014bfd","remote_addr":"[fd00:1122:3344:101::c]:53581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:26.548487976Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"91df6ab7-3f7b-42d5-805b-6a442a0ead34","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8248,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:46.890498102Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52732"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:46.890620255Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"687978a6-fa89-4baa-ab0f-95217cf556b7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52732","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.733688346Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34777"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.733838559Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"feb4ac7e-6246-44ff-bd11-306ec9109714","remote_addr":"[fd00:1122:3344:101::b]:34777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.773412466Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37853"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.77359302Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"03f3e02b-1264-49bd-ab5e-a921b9efa20a","remote_addr":"[fd00:1122:3344:101::b]:37853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.775737799Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.775752709Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"493c3468-42ba-40af-9aa2-85f799d42901","remote_addr":"[fd00:1122:3344:101::b]:34777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.785420826Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"66983d0b-8475-46d2-b220-a154ff8ae24c","remote_addr":"[fd00:1122:3344:101::b]:34777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.837885972Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:32865"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.838059856Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.873268295Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.952471242Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7ce7fd4d-bc30-4249-b9e6-7d073a3c89ed","remote_addr":"[fd00:1122:3344:101::b]:32865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114464,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:49.952877691Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"650d9748-15a1-418c-8b36-9ff368b7ebe7","remote_addr":"[fd00:1122:3344:101::b]:32865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.109003792Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.109151905Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7de5bb68-f055-4b9c-af30-a14d55c708c5","remote_addr":"[fd00:1122:3344:101::a]:52425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.151707189Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.15174073Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ec9e9c0b-cbdf-48c7-8052-7dd01b872b2a","remote_addr":"[fd00:1122:3344:101::a]:52425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.160918746Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cdcd5a97-086a-4f02-88f8-0c50cdef0050","remote_addr":"[fd00:1122:3344:101::a]:52425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.185089578Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33740"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.18517588Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.222895766Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.305036987Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8dfa7162-00d3-4484-bce1-e4a3f9e0a39e","remote_addr":"[fd00:1122:3344:101::a]:33740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119892,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.305249142Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58469"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.305421726Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"133321a7-85bf-47e5-a30a-319b4d65517c","remote_addr":"[fd00:1122:3344:101::a]:33740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.305521538Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"07dfcbcd-6260-4520-bbd4-b2b79fb78bb0","remote_addr":"[fd00:1122:3344:101::a]:58469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.439279648Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36998"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.439421351Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6a1a039f-7754-42a6-ad62-1d14c08fad63","remote_addr":"[fd00:1122:3344:101::c]:36998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.453927936Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.453957327Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1ac1a334-523c-49ea-86b3-0923da7e4c18","remote_addr":"[fd00:1122:3344:101::c]:36998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.496372968Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.496553702Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"43c6a9c6-7b51-4492-b27a-59a1e993046a","remote_addr":"[fd00:1122:3344:101::c]:60599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.505639796Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b7d62faf-e000-49db-9864-e49caa830cf4","remote_addr":"[fd00:1122:3344:101::c]:36998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.725216399Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64661"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.725307352Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.763375405Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.81712924Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dbf0f9a2-325e-4d60-a406-c13c552c7f15","remote_addr":"[fd00:1122:3344:101::c]:64661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91869,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:50.817376376Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8437d6d5-f281-4b8e-91a8-c4dbffa0e3ba","remote_addr":"[fd00:1122:3344:101::c]:64661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:23:56.547054593Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"292da5fc-66a8-46bc-867e-e1f909bb7e5c","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:16.933192543Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:16.933292085Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6f7bafcb-728f-4536-a9aa-c39275485d85","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37036","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.844685208Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.844834591Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"22a525ce-4ffa-4db0-a88a-4b2f2885aad1","remote_addr":"[fd00:1122:3344:101::b]:53313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.900696364Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.900731555Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"78d95ef5-f1b2-48c5-b45b-e3caf8d069d8","remote_addr":"[fd00:1122:3344:101::b]:53313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.912448967Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fbfc9a06-86df-46ca-912f-033a389370f0","remote_addr":"[fd00:1122:3344:101::b]:53313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":228,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.924921737Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:19.925271535Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"a0dee8a7-83cc-478c-8a39-a3e29eeb4768","remote_addr":"[fd00:1122:3344:101::b]:56915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.074766317Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48954"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.074869449Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.112213207Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.16451424Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"05bc81e6-28e3-40b7-b36d-1479a701ebc3","remote_addr":"[fd00:1122:3344:101::b]:48954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89701,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.164842227Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b1fae03f-7b58-49bb-b83a-d820e0a915a8","remote_addr":"[fd00:1122:3344:101::b]:48954","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.259791956Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56705"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.25997164Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ad47fc64-9b65-4f6c-bfa5-91164bfc547d","remote_addr":"[fd00:1122:3344:101::a]:56705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.32327847Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.323314531Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f928def0-3251-4ae1-ae1b-eac5635c62d2","remote_addr":"[fd00:1122:3344:101::a]:56705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.333935969Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"41d4def1-1008-4e03-ab56-0e7b1904dc51","remote_addr":"[fd00:1122:3344:101::a]:56705","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":245,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.501582118Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.501757852Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"13fe461f-3fc8-4730-8949-92c2de69aade","remote_addr":"[fd00:1122:3344:101::a]:39288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.661159306Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50025"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.661279459Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9bd8aea3-b200-4905-bce5-344f6169ca71","remote_addr":"[fd00:1122:3344:101::c]:50025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.673268788Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47246"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.67338681Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.711633868Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.769467775Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"af3e6ae6-6f05-476a-bf33-ee77f498b2e6","remote_addr":"[fd00:1122:3344:101::a]:47246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96128,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.76968959Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50749"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.76969993Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.769732031Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fe28c79f-162d-49e1-9f7c-21ab47c5457d","remote_addr":"[fd00:1122:3344:101::c]:50025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.769926325Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3df5c75e-8c27-4a5a-b43c-e807307c3ff8","remote_addr":"[fd00:1122:3344:101::a]:47246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":189,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.769973636Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0aa9dfeb-8f91-4779-8ae4-d19ea21baedf","remote_addr":"[fd00:1122:3344:101::c]:50749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":178,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.779668214Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"9752b840-a37d-46d7-ba9b-46505811ad99","remote_addr":"[fd00:1122:3344:101::c]:50025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.796048701Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40168"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.796144693Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.82991126Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.885959026Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"877f306e-1cf5-494f-a6f7-b67ff858807a","remote_addr":"[fd00:1122:3344:101::c]:40168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89858,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:20.886183121Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c7d19aaf-8398-4fe0-9f73-7541fe0f5eb2","remote_addr":"[fd00:1122:3344:101::c]:40168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:26.547468536Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"8343d80c-7802-4524-af13-0e0bd430134c","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6971,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:30.117822684Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:46.967683891Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:46.967768613Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"40ed2479-0611-429a-8757-eab3fee29551","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52726","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.725451459Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.725595652Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5fb5a59e-0b58-4bd6-afee-a1bbfd617bb8","remote_addr":"[fd00:1122:3344:101::b]:38935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.758340816Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37074"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.758575892Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"de614b36-f50f-4f1d-b600-6b7e72ca61fc","remote_addr":"[fd00:1122:3344:101::b]:37074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.770747575Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.770769105Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4b9a6c0f-b814-4464-99ae-f2faf33cafe4","remote_addr":"[fd00:1122:3344:101::b]:38935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.780018103Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4012eddb-1734-408b-8e67-52678b24a011","remote_addr":"[fd00:1122:3344:101::b]:38935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":160,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.799718204Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35231"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.799819156Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.836157271Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.901380833Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"4afca120-8594-493c-aca3-a24032162adb","remote_addr":"[fd00:1122:3344:101::b]:35231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101568,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:49.902347345Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7a7a89a5-73c2-452e-90b0-169512cf7d84","remote_addr":"[fd00:1122:3344:101::b]:35231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.117351886Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35298"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.117501819Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3b592511-1ca2-42d4-b82b-472a2535e13c","remote_addr":"[fd00:1122:3344:101::a]:35298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.159707966Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.159738866Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7bdeab2a-db76-46bd-a698-92c567dd3789","remote_addr":"[fd00:1122:3344:101::a]:35298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.174570689Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"66be5885-a182-44ef-8aff-5ddca2144d1a","remote_addr":"[fd00:1122:3344:101::a]:35298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":248,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.250328548Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42560"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.250598174Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"18b5f1b3-768e-4e36-9724-5067315983c8","remote_addr":"[fd00:1122:3344:101::a]:42560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.275217976Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48686"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.275370559Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.31912287Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.38468783Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"22165530-f29b-47e9-89ab-9ada0dc8cd5e","remote_addr":"[fd00:1122:3344:101::a]:48686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109388,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.385043038Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b4982d23-6f3a-4440-8359-22c0711cb5db","remote_addr":"[fd00:1122:3344:101::a]:48686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.401350354Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.401503838Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f2c35acb-c85e-4bde-b80a-d01df046b7ab","remote_addr":"[fd00:1122:3344:101::c]:34177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.417271121Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.417287772Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0ff69091-743b-4793-9a86-f817537656b8","remote_addr":"[fd00:1122:3344:101::c]:34177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.493125772Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.493309636Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4fba6858-5e37-4b44-8e53-18dfd1ccde34","remote_addr":"[fd00:1122:3344:101::c]:35758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.5028646Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8c3b2213-a51e-4aeb-9fca-7e680fbc45ae","remote_addr":"[fd00:1122:3344:101::c]:34177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.595183391Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39148"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.595278793Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.635893454Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.693984536Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dbe8fbb0-0579-448b-a1f7-7c4bd070e4c1","remote_addr":"[fd00:1122:3344:101::c]:39148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98750,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:50.694249102Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8899f9ba-5436-4eb4-98d1-4f5442cc8845","remote_addr":"[fd00:1122:3344:101::c]:39148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:24:56.548185785Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"5d4f3480-0b0a-43a4-b4ea-85a8a512052a","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6975,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:17.000535159Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:17.000630701Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9cbaba8f-38c9-4efc-bd43-4ef25b015b7d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61310","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:19.930952347Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:19.931160182Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ecf02516-196e-4f69-a448-327cc9e8b816","remote_addr":"[fd00:1122:3344:101::b]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:19.960189433Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:19.960240404Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8eaa16b0-13a8-429f-8bec-0a01441b1232","remote_addr":"[fd00:1122:3344:101::b]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:19.977365688Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"461971d4-8154-4d1c-a746-19a0088b069a","remote_addr":"[fd00:1122:3344:101::b]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":252,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.033689861Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:32970"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.033914556Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"662dff22-d903-4b98-86c7-f4222d3537e9","remote_addr":"[fd00:1122:3344:101::b]:32970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.069238388Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59262"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.069335631Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.111501306Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.173403654Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ca43e251-1d4c-490f-a9a4-e12c49c2ef1b","remote_addr":"[fd00:1122:3344:101::b]:59262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.173707351Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"507976ba-fd1e-4349-a6b7-dcd76d20971a","remote_addr":"[fd00:1122:3344:101::b]:59262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.296025014Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59859"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.296239609Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"fc05eae9-088c-4fbc-81be-47b050d4ec08","remote_addr":"[fd00:1122:3344:101::a]:59859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.386441511Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.386667096Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"74c33cdb-95ed-492c-8aa2-afb3775c4cb4","remote_addr":"[fd00:1122:3344:101::a]:46841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.435481441Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.435499601Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"32a2a199-ca14-474d-a363-b307944690c0","remote_addr":"[fd00:1122:3344:101::a]:59859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.444106704Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b1da2949-3989-4a05-bd2d-f90ff8230b57","remote_addr":"[fd00:1122:3344:101::a]:59859","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.5785943Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.578670752Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"cee86817-63a7-4cb9-b92a-a9a7161e2c8a","remote_addr":"[fd00:1122:3344:101::c]:49203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.593258669Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.593271889Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a8700ad1-674c-4d7b-8109-69dc66615e3b","remote_addr":"[fd00:1122:3344:101::c]:49203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.601222688Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2110f28f-cdbd-421a-bd1f-6f46e0564c05","remote_addr":"[fd00:1122:3344:101::c]:49203","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.640916398Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55162"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.641080981Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ce9a2de7-b980-428a-9784-0b38c02dd70d","remote_addr":"[fd00:1122:3344:101::c]:55162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.650459381Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54066"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.650562774Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.690814926Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.749970613Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0291375c-8933-48b3-adf8-ada97e86d2f3","remote_addr":"[fd00:1122:3344:101::a]:54066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99448,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.750223269Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2f1f5c37-78b1-43e7-aaad-ea88c2b4e564","remote_addr":"[fd00:1122:3344:101::a]:54066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.817071238Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37201"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.817242541Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.854432886Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.913616602Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b8667298-94f1-4952-ac46-bc58a8f84bf6","remote_addr":"[fd00:1122:3344:101::c]:37201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96444,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:20.91393751Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9ca9bc41-61b1-4c42-8d7b-69c93e42eca1","remote_addr":"[fd00:1122:3344:101::c]:37201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:26.548447833Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"04efa535-8810-488e-982c-c9595999b020","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8091,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:47.035149798Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:47.035297051Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"c2996054-e672-4a66-b56b-71496832077f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46864","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.655524115Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40655"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.65574418Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9d0c079b-8b8c-499a-a800-b1f3906692bb","remote_addr":"[fd00:1122:3344:101::b]:40655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.712018442Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.712077123Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"61504ae4-8a32-4e6d-9a03-18b95f05221f","remote_addr":"[fd00:1122:3344:101::b]:40655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.720802508Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f9513a17-55b5-4d90-a076-cf0057402371","remote_addr":"[fd00:1122:3344:101::b]:40655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.795180956Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52739"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.795306359Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.835421369Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.894126875Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"023ab612-2a57-4cde-a4e4-43c956ed21d3","remote_addr":"[fd00:1122:3344:101::b]:52739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98838,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.894485163Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55575"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.895057606Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4de60275-a442-4207-b170-d645078ea6ea","remote_addr":"[fd00:1122:3344:101::b]:55575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":437,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:49.895087747Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a56da065-a38f-454f-b551-1aae8e0a8a42","remote_addr":"[fd00:1122:3344:101::b]:52739","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":361,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.12091194Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59207"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.121057084Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"60b44f81-ffc7-4ab2-a583-18579edc3172","remote_addr":"[fd00:1122:3344:101::a]:59207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.135864436Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.135880246Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4656adcf-8a3a-4c76-b7d8-19045aa4845c","remote_addr":"[fd00:1122:3344:101::a]:59207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.144854547Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7afbade5-7b51-450f-8c55-bfbdcf072202","remote_addr":"[fd00:1122:3344:101::a]:59207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.192533717Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.19266968Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"271f8180-96e0-4062-8ab2-3f1b88fd9cb8","remote_addr":"[fd00:1122:3344:101::a]:33020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.270668498Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36664"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.270818102Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.319931703Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.38624797Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"00dfe1f7-4d9b-4d02-b1fc-9aa255beb88d","remote_addr":"[fd00:1122:3344:101::a]:36664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115494,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.386347123Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62635"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.386465605Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"71db2e53-4245-41dc-be16-96a378ac8500","remote_addr":"[fd00:1122:3344:101::c]:62635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.386633679Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1e9e9032-73be-4f94-86bb-fc1e56f85d13","remote_addr":"[fd00:1122:3344:101::a]:36664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.400412678Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.400430648Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"096d7952-5b5b-4828-8110-79a8a0ddf437","remote_addr":"[fd00:1122:3344:101::c]:62635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.409068932Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1b8fe11f-ce77-4862-8c14-ca5fb6fb77d5","remote_addr":"[fd00:1122:3344:101::c]:62635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":230,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.416153851Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53468"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.416247643Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.457944678Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.513250708Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"44fe3347-68f7-41ca-9ffa-6b02e98df659","remote_addr":"[fd00:1122:3344:101::c]:53468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97035,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.513524494Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cb90267d-bf29-4ded-a550-d6e7012325ab","remote_addr":"[fd00:1122:3344:101::c]:53468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.536876398Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59418"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:50.537106443Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"3a8e8b38-4701-417d-9827-390c8d8ae9b9","remote_addr":"[fd00:1122:3344:101::c]:59418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:25:56.548978958Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"94fda7ae-ece2-4cf1-a4fe-7079282927b4","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8396,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:17.06678762Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:17.066886702Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9640d113-e910-485e-934c-b6640011c6e8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52178","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.838082852Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62548"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.838246085Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8c6ceef4-3506-4bef-bb5c-daa55abe9dd5","remote_addr":"[fd00:1122:3344:101::b]:62548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.888123463Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.888152544Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b3ab8f4c-e7bb-4a5c-82d2-f47745aab472","remote_addr":"[fd00:1122:3344:101::b]:62548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.89868943Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2f653563-fbae-4825-895b-7449ee6ae9e2","remote_addr":"[fd00:1122:3344:101::b]:62548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.950345918Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40804"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.950480351Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:19.999364667Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.060603191Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7d0a83b4-f941-4722-b864-1a7993aa5c84","remote_addr":"[fd00:1122:3344:101::b]:40804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.060828316Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46749"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.060989179Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"480ba6bb-3df0-47f3-b790-9aca4aff5265","remote_addr":"[fd00:1122:3344:101::b]:40804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.061162603Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"187159a1-8add-4f66-852d-d37222f85273","remote_addr":"[fd00:1122:3344:101::b]:46749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.195788992Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47506"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.195937865Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"bd94da82-b3a1-4a74-a04b-1b367a5e8f97","remote_addr":"[fd00:1122:3344:101::a]:47506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.210718477Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.210747617Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"326e4c13-a6ef-4e79-ad8b-1577927ce3d4","remote_addr":"[fd00:1122:3344:101::a]:47506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.327123257Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36780"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.327413414Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"3b22244d-8807-42c7-a5a9-d3b1a122ecff","remote_addr":"[fd00:1122:3344:101::a]:36780","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.336965578Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8dcc9dba-b406-46a0-8279-f690907390cb","remote_addr":"[fd00:1122:3344:101::a]:47506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.438567486Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39933"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.438653208Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.479846141Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.542589968Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"024d8f38-b0d2-413c-a73e-e8fd83d32d5d","remote_addr":"[fd00:1122:3344:101::a]:39933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103966,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.542715601Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45259"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.542905895Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"143ae0e8-9270-406d-bf81-01fda185ea3d","remote_addr":"[fd00:1122:3344:101::c]:45259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.54310692Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"370fc273-afcb-494d-b3d4-ffe7b4c0f25c","remote_addr":"[fd00:1122:3344:101::a]:39933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":230,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.557169795Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.557190756Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c557ae5e-d612-4bf2-ad37-571c34c1f0e9","remote_addr":"[fd00:1122:3344:101::c]:45259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.643797698Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.643987422Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5cdf3130-5d9c-4575-ad68-985e7c4000d1","remote_addr":"[fd00:1122:3344:101::c]:47134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.647405269Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"83b5e92b-6ea3-455e-a16b-fa55ea2d5fe2","remote_addr":"[fd00:1122:3344:101::c]:45259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":269,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.715518786Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60924"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.715611788Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.756307591Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.812106022Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ad59cb39-2c7d-4dc8-8712-5a4cc0eb1080","remote_addr":"[fd00:1122:3344:101::c]:60924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96529,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:20.812335457Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ed21a5b2-b1c7-4bb1-b1f2-ba85bf3e1911","remote_addr":"[fd00:1122:3344:101::c]:60924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:26.548352695Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"d17b8346-da19-4f6b-b461-33ea0a9cad47","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7254,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:47.099955398Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:47.10005531Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"871a03f7-f01a-455b-9c7b-594eab5db598","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38459","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.748096787Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65388"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.748244811Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"daacc3ea-67d4-413c-a578-c74edb74f5e2","remote_addr":"[fd00:1122:3344:101::b]:65388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.762797207Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.762824257Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c252d0c9-a7fd-43dd-b340-b4ba2b284392","remote_addr":"[fd00:1122:3344:101::b]:65388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.772094465Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6ca76f25-eb21-4742-8c45-4c395e4ff2ca","remote_addr":"[fd00:1122:3344:101::b]:65388","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.828377767Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53039"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.82849635Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.867964784Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.938300502Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f1cb1dde-c11f-488b-bbaf-57f0318f307e","remote_addr":"[fd00:1122:3344:101::b]:53039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109836,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.938581198Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40895"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.939023988Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0ea54296-bb2e-4434-9ac3-382d54f650b5","remote_addr":"[fd00:1122:3344:101::b]:40895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:49.939037868Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"14ab5eff-c8ef-49e6-b027-1c8cde050659","remote_addr":"[fd00:1122:3344:101::b]:53039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":227,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.091502447Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.09163581Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"07946e9d-24b6-4a4f-9062-c7d3098bcba3","remote_addr":"[fd00:1122:3344:101::a]:48282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.105601543Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.105615703Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"94aea638-2a26-412f-a53d-99a1c53e4c91","remote_addr":"[fd00:1122:3344:101::a]:48282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.115230739Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cc9de5ca-6c2b-452c-ba48-a1b7b5b0a241","remote_addr":"[fd00:1122:3344:101::a]:48282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.220379217Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52685"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.220549251Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7cdd5862-4f2e-4508-a7e6-24be5d1b3106","remote_addr":"[fd00:1122:3344:101::a]:52685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.289466236Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45318"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.289583269Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.332237175Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.392729272Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9f5ce8a8-83af-411f-a79f-7137c41871cd","remote_addr":"[fd00:1122:3344:101::a]:45318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103196,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.393048029Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6163d997-f88f-4eac-8f4c-adff8567f671","remote_addr":"[fd00:1122:3344:101::a]:45318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.42029599Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.420449483Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7fb390e6-7be4-4093-a7e6-68c000b3dc81","remote_addr":"[fd00:1122:3344:101::c]:56834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.463636042Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.463653372Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b1d28970-0627-4389-b3ee-c4e084a52df1","remote_addr":"[fd00:1122:3344:101::c]:56834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.47203505Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0fbd1d26-ee9f-4b0a-85d3-6c4eafdd41f1","remote_addr":"[fd00:1122:3344:101::c]:56834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":224,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.492843546Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.49297604Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8632f728-9849-46e4-995b-19a9e8f0d785","remote_addr":"[fd00:1122:3344:101::c]:33864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.507460334Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57248"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.507561716Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.547602794Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.603704932Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fc7afc03-c19a-49cf-9e99-a16b83470790","remote_addr":"[fd00:1122:3344:101::c]:57248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96187,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:50.603991349Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5587a4ff-7075-4aa6-9e30-8c414ba37fb0","remote_addr":"[fd00:1122:3344:101::c]:57248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:26:56.548599667Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"a580f43f-6f5f-4839-ae52-d694d84f5650","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6989,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:17.134227807Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59379"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:17.134320429Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3ca0c61c-4e70-4273-a4e2-d021c303b237","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59379","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:19.872898836Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47376"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:19.87304628Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"10a2c0ab-1b4b-465f-9b3c-44a2540ae112","remote_addr":"[fd00:1122:3344:101::b]:47376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:19.924254848Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:19.924282699Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5c78f5f2-ddd4-4269-b09c-93e45128229c","remote_addr":"[fd00:1122:3344:101::b]:47376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:19.996750164Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:19.996934488Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0fb4057e-f1aa-453c-a268-adb73035a302","remote_addr":"[fd00:1122:3344:101::b]:49795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.006909711Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"25ba58f9-74f5-4fe8-9927-2cea408274ad","remote_addr":"[fd00:1122:3344:101::b]:47376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":187,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.063618603Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60844"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.063724435Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.106887883Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.167792459Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a634b699-93d6-489d-91c3-c5a727f60aaa","remote_addr":"[fd00:1122:3344:101::b]:60844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.167983013Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48085"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168130256Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"102a492b-fbb6-4de0-81ae-b66594499690","remote_addr":"[fd00:1122:3344:101::b]:60844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168382682Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168442083Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168481804Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168498845Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168512295Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168557686Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168588307Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.168616578Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ff6be876-bdcd-494e-9f28-731814a88b47","remote_addr":"[fd00:1122:3344:101::b]:48085","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":454,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.237780898Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54146"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.237944642Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d50d67db-1f83-47e8-b58b-0916911d52f8","remote_addr":"[fd00:1122:3344:101::a]:54146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.331085611Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.331123631Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ce65b94d-fa19-4abc-92e8-45c025812296","remote_addr":"[fd00:1122:3344:101::a]:54146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.341229078Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"61d9b177-29f1-4509-beec-1a1dcfe21977","remote_addr":"[fd00:1122:3344:101::a]:54146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":265,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.38768065Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.387803293Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"dccab09a-36b4-4bc8-b4db-de11148eb18d","remote_addr":"[fd00:1122:3344:101::c]:46134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.41089565Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51583"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.411027703Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.465116456Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.542989682Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9c8dc44d-be3b-428f-832a-ee622220def3","remote_addr":"[fd00:1122:3344:101::a]:51583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132001,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.54334162Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"dcdf9b85-e9ac-4a1d-be94-b196de87216e","remote_addr":"[fd00:1122:3344:101::a]:51583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":151,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.543404832Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.543422262Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"861975f7-8a1f-4277-8b24-9dfcc72a400b","remote_addr":"[fd00:1122:3344:101::c]:46134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":170,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.552328452Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ffe16391-c4e6-478a-afae-1eb085630c41","remote_addr":"[fd00:1122:3344:101::c]:46134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":332,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.560030855Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.560187968Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ac23c13f-1214-4857-a753-f4ca5cf8afd7","remote_addr":"[fd00:1122:3344:101::a]:50491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.635842055Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54576"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636154462Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636202983Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636242283Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636259134Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636273814Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636313215Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636336466Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.636366096Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"dd26d5e0-bc24-45b8-8c47-6540d8906f37","remote_addr":"[fd00:1122:3344:101::a]:54576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":412,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.663183657Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34868"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.663275019Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.70251509Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.761335518Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5f538b92-af93-4859-9685-08a8703d6421","remote_addr":"[fd00:1122:3344:101::c]:34868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98094,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.761452341Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.761748038Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"18835d09-4d09-44ef-a164-e69b04b5e22c","remote_addr":"[fd00:1122:3344:101::c]:45240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.76185342Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1ddb4e53-4955-41fd-b9a9-8d84bc279c9c","remote_addr":"[fd00:1122:3344:101::c]:34868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798173615Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36405"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798459861Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798508092Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798537443Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798548743Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798555563Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798571353Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798587674Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:20.798604734Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e1c7d1ea-9a6c-4164-9983-4915a44d3f91","remote_addr":"[fd00:1122:3344:101::c]:36405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":350,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:26.550521991Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"1efb1994-9359-4751-9ba3-190d3aeb11e2","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8680,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:47.166570708Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:47.166662691Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"8d5ae568-ae43-4e4d-9d3f-d0e61f7a74c0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55796","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.776644035Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35534"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.776787568Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f1053cbe-9a93-4260-a968-7c6f1998be51","remote_addr":"[fd00:1122:3344:101::b]:35534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.792207694Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.792226434Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b3bd3f1d-2c2c-4d52-b705-e94a37ab3cca","remote_addr":"[fd00:1122:3344:101::b]:35534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.801479182Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5c6da5a1-28a8-4126-92c0-4e78ca8bda88","remote_addr":"[fd00:1122:3344:101::b]:35534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":193,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.847302699Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.847458183Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"043afa39-bf9c-4d6f-a7ac-82ea2334551a","remote_addr":"[fd00:1122:3344:101::b]:53139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.910983997Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65482"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.91115991Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:49.962401939Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.033837891Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"bed22887-88b0-4e6c-99f0-cd4ee623353f","remote_addr":"[fd00:1122:3344:101::b]:65482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122741,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.034180779Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2b1736aa-6d74-4ddd-b596-39753ffeb950","remote_addr":"[fd00:1122:3344:101::b]:65482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.075432334Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45992"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.075583007Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"fa0bbd2b-2b16-4998-b46c-bf17c27da121","remote_addr":"[fd00:1122:3344:101::a]:45992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.141756201Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38890"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.141925925Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.181517463Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.233610351Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"35b54757-bcd3-4bff-a8da-8e4322daeae0","remote_addr":"[fd00:1122:3344:101::a]:38890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91732,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.233860976Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.233913307Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e0c661e2-1410-4df9-a356-5014ead08412","remote_addr":"[fd00:1122:3344:101::a]:45992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.233966359Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53616"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.234246935Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"64a8d831-82d4-4de4-84c0-d4d5d5db72f6","remote_addr":"[fd00:1122:3344:101::a]:38890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":274,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.234387238Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0fd6e2f7-a7cf-47b3-893a-ce38d1d7d136","remote_addr":"[fd00:1122:3344:101::a]:53616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.244552566Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"86fca5af-a7d5-4cc6-a2b8-c2c71b3112e7","remote_addr":"[fd00:1122:3344:101::a]:45992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":262,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.427454557Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33103"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.42757718Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b8576807-5290-45c7-a347-f8af43a7f2ea","remote_addr":"[fd00:1122:3344:101::c]:33103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.452399826Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53788"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.45257921Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c4a77f67-6d93-4cb5-b38a-34a9554f7d16","remote_addr":"[fd00:1122:3344:101::c]:53788","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.468073918Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.468087378Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f3f89b62-4606-4695-bfea-c4b760dfa5b9","remote_addr":"[fd00:1122:3344:101::c]:33103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.476934597Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7d5fc43e-3a47-44fe-ac80-46e186df7088","remote_addr":"[fd00:1122:3344:101::c]:33103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.527216994Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33912"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.527299546Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.567019687Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.623975034Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d2544f02-756b-45a2-adb5-eadbc6003915","remote_addr":"[fd00:1122:3344:101::c]:33912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96702,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:50.62427234Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d9cdfa38-ccb8-457a-bede-c6407126632b","remote_addr":"[fd00:1122:3344:101::c]:33912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:27:56.549794549Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"12088dc7-ca98-49e8-930c-970e4ffc74bb","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7836,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:17.199387357Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:17.199484059Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d39a4dfd-5b4d-45ef-a9ed-e7ca25ebee16","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44044","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:19.952921432Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:19.953105616Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a42ffee9-fb2e-4c03-8a8a-83e557eb73b7","remote_addr":"[fd00:1122:3344:101::b]:50072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:19.968174214Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:19.968190575Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d5706e7d-a78a-4689-913f-8a1d2a6e417e","remote_addr":"[fd00:1122:3344:101::b]:50072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.035907543Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52229"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.036109098Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f899bacf-c379-427b-8023-3f0962f300c6","remote_addr":"[fd00:1122:3344:101::b]:52229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.039875922Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6fb83e55-fa6c-4c3b-a641-5c6084d76942","remote_addr":"[fd00:1122:3344:101::b]:50072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":189,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.060449683Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.060525975Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"905170f4-e224-46ac-b8b6-56e3dad43f95","remote_addr":"[fd00:1122:3344:101::a]:43403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.097695168Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.097725939Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e2057e68-d264-41bd-8d16-c0a7dea49d71","remote_addr":"[fd00:1122:3344:101::a]:43403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.112183504Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4994568f-8492-4220-8989-dae29c44f998","remote_addr":"[fd00:1122:3344:101::a]:43403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":270,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.144203891Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62028"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.144328934Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.191296978Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.266021193Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"608db906-4345-4c25-ae28-cec0afaa0d73","remote_addr":"[fd00:1122:3344:101::b]:62028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121743,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.266663517Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cdbe89cb-f1a0-4439-b887-dc2af55d98f7","remote_addr":"[fd00:1122:3344:101::b]:62028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":226,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.337391924Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.33769659Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"814882fe-c84b-4258-8911-d5b066226153","remote_addr":"[fd00:1122:3344:101::a]:48164","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":219,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.365441233Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33197"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.365587296Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.402892343Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.461832904Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"50540a89-fe7a-4e64-b4d8-190eb162b758","remote_addr":"[fd00:1122:3344:101::a]:33197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96307,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.462126681Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58501"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.462370846Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8ecd87bf-5bba-4f46-9d94-2e5447d1a3e2","remote_addr":"[fd00:1122:3344:101::a]:33197","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":214,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.462450928Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f22ae68c-e76e-400a-907b-5c197b7cef08","remote_addr":"[fd00:1122:3344:101::c]:58501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.521918072Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.521950492Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d0e5d3ce-945e-4b46-be67-e21f69b35b47","remote_addr":"[fd00:1122:3344:101::c]:58501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.532322095Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a1eea63e-b104-4483-895d-9ccb774f6431","remote_addr":"[fd00:1122:3344:101::c]:58501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.661384399Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55516"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.661527232Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"c742203d-3166-414d-95f9-f7db3b150d45","remote_addr":"[fd00:1122:3344:101::c]:55516","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.720957715Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47625"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.721047527Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.759262954Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.814198956Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2e698cc8-3f7c-48de-bb1b-a91d0a2d2039","remote_addr":"[fd00:1122:3344:101::c]:47625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93194,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:20.814498233Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6fec451e-fb85-4f4d-a8c6-4df7e3575a2f","remote_addr":"[fd00:1122:3344:101::c]:47625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:26.550579382Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"c1c7661b-4a8c-4d84-b29d-988933ab8040","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7987,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:47.233256631Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:47.233383345Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"f345ca71-e4de-4040-bbde-d0b9d64dd5f3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42654","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.741366902Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.741515545Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"309b2bca-0569-4da9-bd0f-80b8a9d9fbf4","remote_addr":"[fd00:1122:3344:101::b]:48345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.766180068Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.766359422Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"69007c1b-6a8e-4977-b0a4-5d82fb5b9035","remote_addr":"[fd00:1122:3344:101::b]:53384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.808962187Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.808980688Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"525bef28-bc18-4331-8446-6b275fa79a0a","remote_addr":"[fd00:1122:3344:101::b]:48345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.818103672Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d44894e0-c187-46a3-823c-69f2e41b3843","remote_addr":"[fd00:1122:3344:101::b]:48345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":201,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.835432471Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63044"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.835517933Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.872008981Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.950651264Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"18980277-4686-4e6b-a2eb-7ca0acbe9831","remote_addr":"[fd00:1122:3344:101::b]:63044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115160,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:49.951045743Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f2eaaea8-8cc3-4933-8f64-dab1208ce92b","remote_addr":"[fd00:1122:3344:101::b]:63044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.089370105Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51873"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.089537749Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0f9e42e8-2b8b-4e9a-9403-b1d309e4417d","remote_addr":"[fd00:1122:3344:101::a]:51873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.133933754Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.133950044Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"19f199a0-abde-495f-91b4-db13857daa1b","remote_addr":"[fd00:1122:3344:101::a]:51873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.142273221Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d96a9146-07f2-4536-9d7b-5b14895e1566","remote_addr":"[fd00:1122:3344:101::a]:51873","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":318,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.174412062Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.174592126Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"bb0fd686-3064-446d-9f45-d906a7a1f232","remote_addr":"[fd00:1122:3344:101::a]:42673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.190197026Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50952"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.190294388Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.234410047Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.310454952Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1a363fac-d563-4ff8-b804-d0f82b5866d9","remote_addr":"[fd00:1122:3344:101::a]:50952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120186,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.310862431Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9a43ecba-8e0d-49dc-9e94-2399a695e847","remote_addr":"[fd00:1122:3344:101::a]:50952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.393001423Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.393152437Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f6798aa5-d3b0-47e2-8ff9-9314a954c97f","remote_addr":"[fd00:1122:3344:101::c]:45465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.437481531Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.437507761Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c60d6f86-0edc-4186-810c-30d2b64d50ea","remote_addr":"[fd00:1122:3344:101::c]:45465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.446001432Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8c810be1-7fa1-45cc-ab68-8d725ae98b50","remote_addr":"[fd00:1122:3344:101::c]:45465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.532498231Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.532644074Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"6001b3bb-0c89-4183-bdff-a67436d4f93f","remote_addr":"[fd00:1122:3344:101::c]:37597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.593714094Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47262"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.593855687Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.62878227Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.681420271Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"11f5b99c-a4bd-4fe0-bd28-54a8f71c7fc4","remote_addr":"[fd00:1122:3344:101::c]:47262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87622,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:50.681730708Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ff3cb8a1-7406-43e3-a816-d2ad5b52f9eb","remote_addr":"[fd00:1122:3344:101::c]:47262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:28:56.550717059Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"4722aed0-6a8b-463e-bcf6-97d716989453","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8540,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:15.738448039Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:15.738531761Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:15.771520061Z","hostname":"buskin","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999995689s","next_cleanup":"Instant { tv_sec: 1855, tv_nsec: 51922470 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:17.265892889Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49731"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:17.266014821Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"92baf5d5-4ddb-4d92-b27a-7f215493aa46","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49731","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.008427015Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43921"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.008594418Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3ee29ec7-6f6d-4347-a2fe-3dbf41661178","remote_addr":"[fd00:1122:3344:101::b]:43921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.036451343Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36064"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.036712399Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"24c0ec36-8316-4ebf-8f8a-ae4bba6355fd","remote_addr":"[fd00:1122:3344:101::b]:36064","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.062859345Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.062876046Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"53a21537-6d90-4982-af66-94d128d08f89","remote_addr":"[fd00:1122:3344:101::b]:43921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.074821314Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1c4a6802-2916-4439-8750-edfa586e03d6","remote_addr":"[fd00:1122:3344:101::b]:43921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.133294745Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53148"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.133391007Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.172265508Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.232618432Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e298c8b1-49c9-4512-ae8d-31053a992b7d","remote_addr":"[fd00:1122:3344:101::b]:53148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99243,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.233108553Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4e079107-d463-4e05-bdef-607eea4476f3","remote_addr":"[fd00:1122:3344:101::b]:53148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.293678221Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.293824104Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e1b37acf-628c-481b-b061-26aed833cdb1","remote_addr":"[fd00:1122:3344:101::a]:41084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.316672097Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.316702957Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e6573b91-6b25-42b1-bcce-87afb6c2c7d8","remote_addr":"[fd00:1122:3344:101::a]:41084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.393365686Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.393644573Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"eb013ceb-3c97-43cc-ab45-4b52bb769290","remote_addr":"[fd00:1122:3344:101::a]:50480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.409806745Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e725fa0e-0a47-4353-9c7d-01699b177295","remote_addr":"[fd00:1122:3344:101::a]:41084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.43812388Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37575"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.438201752Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.476738766Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.534506791Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"bc2507e7-53c4-4c2e-b4d8-47f5f6942ff5","remote_addr":"[fd00:1122:3344:101::a]:37575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96334,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.534643694Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65114"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.534754437Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"29f030b7-f0d4-453b-ab57-3c2d520af1c2","remote_addr":"[fd00:1122:3344:101::c]:65114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.53490853Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"aaaf95a4-ea60-4e73-892a-a3fc879f087d","remote_addr":"[fd00:1122:3344:101::a]:37575","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.577227289Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.57724781Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3e6b1d4b-591c-4665-b974-c121151790bc","remote_addr":"[fd00:1122:3344:101::c]:65114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.649735595Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48938"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.649937Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"56b0ef91-e0b4-4c4f-9fd9-95e68c05cd6a","remote_addr":"[fd00:1122:3344:101::c]:48938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.654632895Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"9d23591b-5590-41f3-afa5-6189660c4429","remote_addr":"[fd00:1122:3344:101::c]:65114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.729425782Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54299"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.729525435Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.76899954Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.824415972Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"57f573e0-503e-485c-96eb-d17d97984d67","remote_addr":"[fd00:1122:3344:101::c]:54299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94925,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:20.824659818Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4eeb53fb-88da-4417-829a-ed1f1e3730ae","remote_addr":"[fd00:1122:3344:101::c]:54299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:26.55050952Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"36429d89-bc4a-48c8-9a0a-98a9f4034cb5","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8194,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:30.121100043Z","hostname":"buskin","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:47.298203559Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43321"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:47.298311191Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"2bbc4e7f-b5bb-45cb-aaea-476dda2ff66e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43321","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":26,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.887849106Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.888126383Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5a1071f9-bb4f-4b04-919c-74c8b5e51944","remote_addr":"[fd00:1122:3344:101::b]:59704","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.894364432Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40605"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.894471395Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6cc4ff01-ce04-4ef3-a459-21f81fa29d28","remote_addr":"[fd00:1122:3344:101::b]:40605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.959534584Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.959586985Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ea042a2c-7972-4133-a5d4-a5198738c3d2","remote_addr":"[fd00:1122:3344:101::b]:40605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:49.971171725Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"545e767b-1847-4fcc-abfc-49df1d56de4a","remote_addr":"[fd00:1122:3344:101::b]:40605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.06193212Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55163"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.062061113Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.099087733Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.151066019Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ca92b11d-2705-4596-a02c-b0b50c777cc3","remote_addr":"[fd00:1122:3344:101::b]:55163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89054,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.151209372Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54713"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.151395616Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"248531a9-c4b3-4bd0-87f7-d764d061ca7e","remote_addr":"[fd00:1122:3344:101::a]:54713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.151510959Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0ddd8aec-59f8-4ccc-8120-79e0ee267c9d","remote_addr":"[fd00:1122:3344:101::b]:55163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":224,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.16405362Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.164235434Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"43a84be9-e9e2-46ba-91e2-990cc93c546c","remote_addr":"[fd00:1122:3344:101::a]:48339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.166876753Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.166889303Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7012d652-f0f3-444c-a6d4-4fb2816264c0","remote_addr":"[fd00:1122:3344:101::a]:54713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.176392477Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0e1a0a6c-7cca-4c72-8386-66be44fc255c","remote_addr":"[fd00:1122:3344:101::a]:54713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.256876731Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61046"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.257027295Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.304304415Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.367824769Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"670f9451-1db6-43d2-9f83-0be40aaa7d11","remote_addr":"[fd00:1122:3344:101::a]:61046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110866,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.368151896Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0c75aaeb-cd3d-42fe-9313-7fb21d04b1f5","remote_addr":"[fd00:1122:3344:101::a]:61046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.393469744Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.393616427Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ef2e05c2-9744-4148-ba55-d85507e06d4a","remote_addr":"[fd00:1122:3344:101::c]:35427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.43607934Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.436120281Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"82f3d809-7e69-438a-9b63-b759e6c04929","remote_addr":"[fd00:1122:3344:101::c]:35427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.444432267Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b352e296-3686-4744-abeb-2e76469426b6","remote_addr":"[fd00:1122:3344:101::c]:35427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.582074644Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43433"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.582176406Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.618549271Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.673115055Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8579b859-c690-4e75-b12a-2a357d70f39f","remote_addr":"[fd00:1122:3344:101::c]:43433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90979,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.67334781Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.673465433Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"bbc85ab8-2bc3-4a53-b747-66587ea75331","remote_addr":"[fd00:1122:3344:101::c]:43433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:50.673635237Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e6c562b2-41e3-4e24-a371-c7c0845b2587","remote_addr":"[fd00:1122:3344:101::c]:46787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:29:56.55078444Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"705e16f3-c537-4e2f-beac-f7c49f468005","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:17.330138018Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52803"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:17.33023493Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b3987174-126b-4d7f-9fe7-be1770bfc1ad","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52803","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:19.898202291Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39210"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:19.898431466Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"e2a08c6e-8578-4c62-aa3e-1a755003413c","remote_addr":"[fd00:1122:3344:101::b]:39210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:19.932856108Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34686"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:19.932975621Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"acd94906-a265-418a-90d9-dcb8f482e5e6","remote_addr":"[fd00:1122:3344:101::b]:34686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.014682473Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.014706963Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4ee0bea8-7f99-4aef-abed-5737f5e6c365","remote_addr":"[fd00:1122:3344:101::b]:34686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.026471977Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5df3ea25-b700-4243-a234-8cc30dbadcea","remote_addr":"[fd00:1122:3344:101::b]:34686","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.153735071Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56922"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.153836413Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.192882749Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.270319645Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"36d938e8-df95-479d-a16f-2dc5c0a72a23","remote_addr":"[fd00:1122:3344:101::b]:56922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116512,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.270607232Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.270801646Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"97ab2a5e-2f02-43cd-b3ec-24854f33c92a","remote_addr":"[fd00:1122:3344:101::b]:56922","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.271033221Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"d05238ad-280a-45f9-905c-4779530f3a64","remote_addr":"[fd00:1122:3344:101::a]:46537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":253,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.349114042Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.350536764Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"222b1919-5d91-46a0-aa6c-4ddf3c390de9","remote_addr":"[fd00:1122:3344:101::a]:42386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.370356148Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.370392359Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2fc8b660-eca1-40f7-9322-08ff7cf9ae20","remote_addr":"[fd00:1122:3344:101::a]:42386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.380435984Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6df74707-074e-4ca7-b42c-7a4d5c0e16ac","remote_addr":"[fd00:1122:3344:101::a]:42386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":317,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.437379541Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64988"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.437489374Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.47566734Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.530403917Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"c30e5609-a530-4b00-95f5-250cf0dce9e8","remote_addr":"[fd00:1122:3344:101::a]:64988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92956,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.530630932Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.530720724Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cac8cd9f-4b5d-4d2f-91e2-93a80822df12","remote_addr":"[fd00:1122:3344:101::a]:64988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":137,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.530832367Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ecfd09b8-0613-4b24-b349-8d1ba196e749","remote_addr":"[fd00:1122:3344:101::c]:53694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.573066534Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.573084404Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0cc0c481-e76a-4230-890d-0255ea1e9958","remote_addr":"[fd00:1122:3344:101::c]:53694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.58226915Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"020a267c-fad3-4dd3-92ba-ae59d214b189","remote_addr":"[fd00:1122:3344:101::c]:53694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":305,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.77268871Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46153"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.772892825Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"8aeae196-faae-424f-b451-41a4c79bff43","remote_addr":"[fd00:1122:3344:101::c]:46153","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.820754508Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44503"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.820869071Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.857803389Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.913512737Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e454a238-0f8f-447e-a516-866f021b166a","remote_addr":"[fd00:1122:3344:101::c]:44503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92687,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:20.913780123Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"73ff03f9-2622-43b5-8126-04643b0658d2","remote_addr":"[fd00:1122:3344:101::c]:44503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:26.551443647Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"b6df0a86-8255-4a87-adbf-9e710aeef8ba","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8495,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:47.363328324Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:47.363458137Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"cbea086b-43cb-4c3d-b393-dd766b80537e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63300","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.711915698Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.712092382Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d222fe06-1d7a-4bbe-9052-0fd5d688e23a","remote_addr":"[fd00:1122:3344:101::b]:44310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.751563416Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.751582017Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7c633b25-b28c-4adb-8b1a-3b64f978b12b","remote_addr":"[fd00:1122:3344:101::b]:44310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.757823567Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.758015961Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"158eed12-7e93-4fbf-a898-f9db05a820bf","remote_addr":"[fd00:1122:3344:101::b]:53759","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.760921186Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b1e7ce57-32dc-4341-a52f-c3a056c0aade","remote_addr":"[fd00:1122:3344:101::b]:44310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.792952115Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65053"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.793038077Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.831317975Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.887192617Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b468abb5-60c9-4407-bc9c-a2c18eeff555","remote_addr":"[fd00:1122:3344:101::b]:65053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94184,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:49.887637697Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cec0564e-caa2-44a2-8ee6-1885bd10145b","remote_addr":"[fd00:1122:3344:101::b]:65053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.091685813Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.091836356Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"90c3c44a-4a4e-4412-82b4-8e523bc289d5","remote_addr":"[fd00:1122:3344:101::a]:37639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.113329608Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.113552983Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"ad130d04-2acb-49ed-a88a-c389981d3926","remote_addr":"[fd00:1122:3344:101::a]:39134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.132716122Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.132745413Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e3923eca-8c38-474b-9e33-8b2b4f9aa855","remote_addr":"[fd00:1122:3344:101::a]:37639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.141361867Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"81e78d7b-a8e3-43da-b28c-58068f5dca68","remote_addr":"[fd00:1122:3344:101::a]:37639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.176142556Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33618"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.176226148Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.217824431Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.293755904Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7f4ff611-e86c-428c-ac91-8e9f38c94585","remote_addr":"[fd00:1122:3344:101::a]:33618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117550,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.294236444Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"427d2e4c-abc7-4e70-8dd8-e1c09b8a3e21","remote_addr":"[fd00:1122:3344:101::a]:33618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.43316749Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.433304013Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d32ca091-2fe4-4a5a-a4f5-f83c457bf044","remote_addr":"[fd00:1122:3344:101::c]:38071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.447949571Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.447966742Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"487136de-e144-429c-b3b6-b075153dbea6","remote_addr":"[fd00:1122:3344:101::c]:38071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.475816626Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.47598285Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"f8632c4d-3fa4-449f-848b-50569ed74469","remote_addr":"[fd00:1122:3344:101::c]:60981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.478851414Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0b3b4265-d587-4302-8b18-79b5fe760210","remote_addr":"[fd00:1122:3344:101::c]:38071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.594773454Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40454"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.594870686Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.634201068Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.691132254Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"756a10cf-e443-4cd1-b818-aac42793e3bf","remote_addr":"[fd00:1122:3344:101::c]:40454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96301,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:50.69140196Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ae0be170-de3b-496a-9932-b18730e1661a","remote_addr":"[fd00:1122:3344:101::c]:40454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:30:56.55048308Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"243e6904-4cbc-4b87-a78c-50587979d7a8","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7470,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:17.397758043Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:17.397855005Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"928b7ce7-bb84-44f8-b803-2df7511fe12e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63313","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.839539905Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61365"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.839683019Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"40341282-0428-4a12-a853-849710519bcc","remote_addr":"[fd00:1122:3344:101::b]:61365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.880504723Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.880525504Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9a3e67b2-d36a-43d2-b68a-b8fc581f61a5","remote_addr":"[fd00:1122:3344:101::b]:61365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.894724852Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4ed01c8c-3091-437b-a437-c47c0fa48f66","remote_addr":"[fd00:1122:3344:101::b]:61365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":213,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.996349691Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:19.996621987Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2f430adf-2303-47df-b109-bc9522ff0dfa","remote_addr":"[fd00:1122:3344:101::b]:45432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.02393609Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56566"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.024046002Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.060398747Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.116198008Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d1814e5c-0bc6-4370-9b24-963a664d4811","remote_addr":"[fd00:1122:3344:101::b]:56566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92197,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.116514516Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7b9a4be5-9f12-44f3-b9bf-266b05091ca5","remote_addr":"[fd00:1122:3344:101::b]:56566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.116594368Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.116764491Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"dd7d93c1-db0e-40a5-b0fd-e84291b766b0","remote_addr":"[fd00:1122:3344:101::a]:33540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.189500162Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.189533953Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"aa251719-5816-434f-b447-101a5697a84b","remote_addr":"[fd00:1122:3344:101::a]:33540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.204901898Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"afe3513e-b374-4d33-ae7c-722a3107ccae","remote_addr":"[fd00:1122:3344:101::a]:33540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":363,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.337110412Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39909"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.337220295Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.377710753Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.434748402Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"657879af-3d7c-4392-89ab-76f40fd73be1","remote_addr":"[fd00:1122:3344:101::a]:39909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97569,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.434961407Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.435141931Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7703a31c-5c87-4e68-9ce8-97628450f454","remote_addr":"[fd00:1122:3344:101::a]:39909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.435306134Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"639c112f-236d-409a-a6e8-3c05322e625f","remote_addr":"[fd00:1122:3344:101::a]:51738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":217,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.602320869Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.602544684Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"17d25906-aac7-4ea6-b9d7-fe5a3df7cca7","remote_addr":"[fd00:1122:3344:101::c]:46684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.657422025Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.657535777Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c9f59864-9303-4088-ab87-c0eb320cbd2d","remote_addr":"[fd00:1122:3344:101::c]:57020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.672498333Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.672514013Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"585efacc-9300-4cba-bcb1-830e1830416c","remote_addr":"[fd00:1122:3344:101::c]:57020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.681104606Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2ab6c8ee-d7e2-427b-8457-71a7c8aa9344","remote_addr":"[fd00:1122:3344:101::c]:57020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.718209408Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33902"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.71830052Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.75709655Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.813028904Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6f6181ee-7c24-4e7a-9999-4e57565246a3","remote_addr":"[fd00:1122:3344:101::c]:33902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94759,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:20.813314071Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"58e43426-d300-48f6-9dcc-8c3188f2cca7","remote_addr":"[fd00:1122:3344:101::c]:33902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:26.551248983Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"c33baa5e-5bcf-45a1-86e8-7051d28c44c9","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7609,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:47.430627854Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:47.430719816Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"19e97c26-d7c9-469a-ad0e-64ba8406c0c4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55915","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.782930209Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.783087293Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0defe6c2-7543-4bd0-b552-e1bc8956b08f","remote_addr":"[fd00:1122:3344:101::b]:52107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.823335986Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.823362286Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0a25bb06-ba21-4205-9f9a-e2bd3b9b9d3f","remote_addr":"[fd00:1122:3344:101::b]:52107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.833411092Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7bf8c899-0d13-4ab5-b27b-66f99e599f0b","remote_addr":"[fd00:1122:3344:101::b]:52107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":209,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.863434275Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.863590508Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5a91748d-411a-4444-946b-c39ba5f65124","remote_addr":"[fd00:1122:3344:101::b]:47622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.91366686Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53176"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.913807763Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:49.962053935Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.028443804Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3745b1ca-37d5-4d55-b166-2add6afce7a5","remote_addr":"[fd00:1122:3344:101::b]:53176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114698,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.028812792Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"06a03005-5dba-4ab1-b9e6-90053147decb","remote_addr":"[fd00:1122:3344:101::b]:53176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.072363859Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.072524173Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"19726cac-69ca-49ba-9e03-dfc12103a2b9","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.125721735Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.125743686Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d96d2f77-2b35-4ba2-9ed8-d5e6090f3ae0","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.135037574Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0bb520a5-4e3a-448e-b063-c13859cdddd1","remote_addr":"[fd00:1122:3344:101::a]:39770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":297,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.219856386Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44223"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.219996469Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.266044332Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.33331092Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"ffb44bb9-8b97-4387-8a8f-a5ee5fcd3ec4","remote_addr":"[fd00:1122:3344:101::a]:44223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113377,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.333568866Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49653"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.333657578Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0984bc59-e5e5-47b1-8b87-ef8a11fdcdde","remote_addr":"[fd00:1122:3344:101::a]:44223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.333851753Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"7d8b85f0-cfb0-4f61-a2da-8611ed0fcb21","remote_addr":"[fd00:1122:3344:101::a]:49653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.356167143Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.356299636Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9c26814e-5012-4296-a130-ef8a05998619","remote_addr":"[fd00:1122:3344:101::c]:33863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.403159067Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.403189517Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"22c8dda1-c0bb-4977-9a74-f2e4972aa740","remote_addr":"[fd00:1122:3344:101::c]:33863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.41267171Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7a7d2401-d7ca-4d99-bc0d-6b26988ecef1","remote_addr":"[fd00:1122:3344:101::c]:33863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":280,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.527600387Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.527754521Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"60bcddf0-42a4-44b8-b9a0-9c4efe389404","remote_addr":"[fd00:1122:3344:101::c]:44150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.568352601Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34400"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.568439643Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.608068141Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.663856072Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"4c5748fc-4305-41c4-bd96-49512ac07cab","remote_addr":"[fd00:1122:3344:101::c]:34400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95461,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:50.664147959Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d99f623d-44d9-4539-a3fc-4b8aaaa921ec","remote_addr":"[fd00:1122:3344:101::c]:34400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:31:56.550396587Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"0c909ab8-9a9e-4cf3-a077-b799f42a1589","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7458,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:17.463679468Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:17.463861942Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7d53ab18-1c2d-4427-8e5e-39c67ab01f61","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44100","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:19.756793037Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:19.757008822Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"10d6cb20-8590-43a4-af5f-6e1c5216b560","remote_addr":"[fd00:1122:3344:101::b]:47208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:19.831054642Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:19.831087553Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e5367e2c-7956-4f89-9838-2253cc28500c","remote_addr":"[fd00:1122:3344:101::b]:47208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:19.843054251Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5c140e71-520e-4993-b188-d80765c16b64","remote_addr":"[fd00:1122:3344:101::b]:47208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":224,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.141602585Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63322"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.141779029Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"eefe9424-425a-41e2-a1de-41a46c8c0364","remote_addr":"[fd00:1122:3344:101::b]:63322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.208474785Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61199"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.208560637Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.208572017Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.208630968Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.21046022Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.216465174Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61866"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.216842172Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"03bf1501-ef5f-40b7-8f50-563288938405","remote_addr":"[fd00:1122:3344:101::a]:61866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":183,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.224522285Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.224560186Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.243416139Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42058"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.243771716Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.277939983Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.278021615Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.308832315Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.315827417Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.316453719Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"79eb6066-1e80-4c42-b42d-239c2bb78c68","remote_addr":"[fd00:1122:3344:101::a]:61866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7999,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.324773863Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.325278634Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.340498325Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8183b402-ab0e-494b-b7a6-633e2f7b1710","remote_addr":"[fd00:1122:3344:101::a]:61866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1316,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.365505466Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.365584108Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.38886852Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40328"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389426782Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389500144Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389549925Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389574176Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389600477Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389663578Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389691459Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.389715369Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"bf1555c2-367e-4d2e-82d1-97574880b147","remote_addr":"[fd00:1122:3344:101::b]:40328","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":632,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.408684174Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.408751436Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.418868093Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b013594b-29f0-419f-ad9f-a039563e5304","remote_addr":"[fd00:1122:3344:101::b]:42058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175279,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.419371454Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f0513bec-ebc9-4edc-b34e-afbb5a621df7","remote_addr":"[fd00:1122:3344:101::b]:42058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":221,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.441690425Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.441753796Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.467013752Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.467076004Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.490010828Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.49008642Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.493494907Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60250"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.501938556Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.501954946Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.502071739Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.506795314Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.514542468Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.514599009Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.52623321Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.526267231Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.542759881Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"aa2ddc83-e666-439b-96d2-4bf2f414f5d2","remote_addr":"[fd00:1122:3344:101::b]:61199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":334202,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.548249374Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.548263564Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.560233083Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65124"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.560500259Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"b98d8dbb-862f-44f4-8b82-2b263d528f0e","remote_addr":"[fd00:1122:3344:101::a]:65124","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.561045881Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.561172464Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2b407063-6522-4e83-9416-21feb7950e2d","remote_addr":"[fd00:1122:3344:101::c]:34545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.579494795Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.579555616Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.584606259Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.584681851Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"aeaa8726-eec3-41c5-81c9-8dce24e7492d","remote_addr":"[fd00:1122:3344:101::c]:34545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":722,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.595647827Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b13c3132-77f0-47ce-97c7-b555da946bd2","remote_addr":"[fd00:1122:3344:101::c]:34545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":350,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.602322827Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37015"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.602525321Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.610472749Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.610529291Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.643897629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.64396405Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.651252684Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.673594245Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.673658416Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.685171404Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58476"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.685725217Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.685815829Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.68588039Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.685933731Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.685979722Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.685999763Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.686049824Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.686083915Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ade770cf-6585-402e-9470-cd5acfab9a1f","remote_addr":"[fd00:1122:3344:101::a]:58476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":716,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.700378135Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.700469367Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.723085535Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b53fd909-b88e-4114-a9a1-19641e37664c","remote_addr":"[fd00:1122:3344:101::a]:37015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120636,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.723435732Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"034e5011-9de2-4781-8658-5559206a0f40","remote_addr":"[fd00:1122:3344:101::a]:37015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.725175011Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.725387566Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"62271731-dd8d-4ca2-8361-71f2419f947e","remote_addr":"[fd00:1122:3344:101::c]:59044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.725665542Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.725704863Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.731729898Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54474"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.740230639Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.740261589Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/648bb898-81e2-4bc5-92c0-fede81914617/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.740342642Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.745576889Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.754757865Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.754802356Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 9d337a02-8b44-4fe3-b02d-dffc8c87f032 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.769540726Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.769583877Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_9d337a02-8b44-4fe3-b02d-dffc8c87f032"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.775004859Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55993"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.775468529Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.796779417Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"cd7b4d16-8f0e-470a-9dbb-3b1aac5e01d8","remote_addr":"[fd00:1122:3344:101::a]:60250","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":303037,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: fd3cb939-7a27-4242-9684-f71fce5b8238 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.799107629Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.79911946Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_fd3cb939-7a27-4242-9684-f71fce5b8238"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.820071449Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b35cb2aa-7e4b-4820-82a1-1376e0cb1176 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.828539759Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.828596641Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b35cb2aa-7e4b-4820-82a1-1376e0cb1176"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.849688983Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57879"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850301967Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2d75a81e-7e26-4d89-8f45-4a8a30dcc43e, kind: Service { id: 065cea45-0eca-4287-90a0-2c3c20ec32f8 }, name: Name(\"external-dns-065cea45-0eca-4287-90a0-2c3c20ec32f8\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 237, 64])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: fe340e0e-7e73-4e4d-9e13-6fa58166803a, kind: Service { id: b24cb173-c3a3-4237-abe6-0e7953a415e5 }, name: Name(\"external-dns-b24cb173-c3a3-4237-abe6-0e7953a415e5\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 178, 2])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3cdfaf5a-6ab2-4340-badd-90537467f215, kind: Service { id: 1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad }, name: Name(\"nexus-1ab9e34a-bb03-4bfb-a926-0dc397c4d4ad\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 196])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 7ca6d922-6012-48ca-ab83-904f1e9b7f23, kind: Service { id: 0a5aaad3-b9af-4668-92b2-6758e325a2ed }, name: Name(\"nexus-0a5aaad3-b9af-4668-92b2-6758e325a2ed\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c6bf8c6b-87e0-4464-b6f8-ef452a4011b6, kind: Service { id: 12bea37a-3ccb-4c9f-8c92-9d6095867a0f }, name: Name(\"nexus-12bea37a-3ccb-4c9f-8c92-9d6095867a0f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850369359Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.85042209Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850461191Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850475561Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850521882Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850547183Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.850582933Z","hostname":"buskin","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a72cbf61-e22a-4aeb-8340-230f3b22b7ff","remote_addr":"[fd00:1122:3344:101::c]:57879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":688,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 7d8d774a-5a14-4ea7-9916-6f77f3cc41c0 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.854188444Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.854205895Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_7d8d774a-5a14-4ea7-9916-6f77f3cc41c0"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dfc55466-0ce1-427c-805f-4acd9ba37666 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.878809955Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.878886157Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_dfc55466-0ce1-427c-805f-4acd9ba37666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.885188438Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0f5167c8-0bef-4cfe-82f8-6dd7089b8730","remote_addr":"[fd00:1122:3344:101::c]:55993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109947,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.885565517Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3fa3af26-837a-484b-98eb-1797e49f3513","remote_addr":"[fd00:1122:3344:101::c]:55993","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: e507db4a-2c57-4f91-87c2-2f1d428e74d8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.902749022Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.902804624Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_e507db4a-2c57-4f91-87c2-2f1d428e74d8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b7548ed7-f361-44b2-a96e-7c703af98968 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.926125027Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.926175397Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b7548ed7-f361-44b2-a96e-7c703af98968"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 715f86b6-8047-4de8-af26-ede400e16cd7 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.947851414Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.947903315Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_715f86b6-8047-4de8-af26-ede400e16cd7"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 305d4c56-8d7f-48cd-9171-b2ebd9a8936c (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.969845807Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.969901008Z","hostname":"buskin","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_305d4c56-8d7f-48cd-9171-b2ebd9a8936c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:20.982995092Z","hostname":"buskin","pid":1566,"uri":"/inventory","method":"GET","req_id":"795fcaa4-2f1b-4f60-8a4f-c034c98acd6b","remote_addr":"[fd00:1122:3344:101::c]:54474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251181,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:26.55215489Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"0121cab0-19a0-4f39-ad41-84b2a62cf878","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8301,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:47.496891378Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54780"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:47.49698769Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"fc4e83d7-dd1e-4c03-bf0a-3dc9d8905252","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54780","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.787606624Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53987"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.787769767Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"34b8ff6c-6b9d-4125-9d37-a69a9065efda","remote_addr":"[fd00:1122:3344:101::b]:53987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.802669312Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.802691632Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1e2740d5-809b-41e2-90cd-34477870843a","remote_addr":"[fd00:1122:3344:101::b]:53987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.812004801Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"73465ef8-04ce-4ee2-abf9-29244a433f1e","remote_addr":"[fd00:1122:3344:101::b]:53987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.957468472Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58771"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.957654926Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"5128fa3a-abd0-424e-9fb4-00bbd13b8bd1","remote_addr":"[fd00:1122:3344:101::b]:58771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.992262742Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40230"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:49.992443186Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.036378181Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.091705712Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"38616fc5-968f-4f22-8737-e74bfce112f2","remote_addr":"[fd00:1122:3344:101::b]:40230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99351,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.092028779Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ffa38e82-008b-4d8e-b9df-999b31344890","remote_addr":"[fd00:1122:3344:101::b]:40230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.111339832Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.111484245Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"095faede-dc1c-4e7f-bcb9-68b58ef85799","remote_addr":"[fd00:1122:3344:101::a]:46451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.123148177Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37584"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.123381892Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"750fe886-b63c-48b0-89e8-fc607db2c16e","remote_addr":"[fd00:1122:3344:101::a]:37584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.165042576Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.165061667Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"96790a7e-e648-4f23-ae6f-8e2fb6b8092d","remote_addr":"[fd00:1122:3344:101::a]:46451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.174210842Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"be114d33-a73b-4c88-a5cb-3f074152b79a","remote_addr":"[fd00:1122:3344:101::a]:46451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.18616348Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53294"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.186256272Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.227494377Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.302455488Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5f8a415a-e31f-41f0-ac57-c7bb1744903b","remote_addr":"[fd00:1122:3344:101::a]:53294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116235,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.302815786Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"54f9c684-bc7a-4189-92ad-0738c3f68cea","remote_addr":"[fd00:1122:3344:101::a]:53294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.365535022Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35752"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.365684976Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"64a58219-1132-4ee1-a38e-52cdab192de6","remote_addr":"[fd00:1122:3344:101::c]:35752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.379630148Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.379655809Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"63e20915-8fc9-4b13-919a-cecaa2a5c6ef","remote_addr":"[fd00:1122:3344:101::c]:35752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.388855335Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1d1f023e-8a65-45e9-ac94-b5af3f8b6c9d","remote_addr":"[fd00:1122:3344:101::c]:35752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":278,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.494016633Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42076"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.494179307Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4e1bc9f8-d9d4-4f25-b3b4-91e54a3660f1","remote_addr":"[fd00:1122:3344:101::c]:42076","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.667536094Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53592"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.667635827Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.706705632Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.764484728Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8046611b-61d7-4c5f-a652-a08774e09ed9","remote_addr":"[fd00:1122:3344:101::c]:53592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96894,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:50.764739124Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c1957880-cc21-48d8-9359-198173338d33","remote_addr":"[fd00:1122:3344:101::c]:53592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:32:56.551279865Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"320ef74e-c2c0-4c05-bc80-c14e6932cea3","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8011,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:17.531066Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61255"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:17.531160773Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"ff1b1d75-eedd-4726-bc45-7365e50e5d00","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61255","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:19.955589835Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:19.955784849Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"433ce9d5-1772-47fd-988c-38a39c56ac8f","remote_addr":"[fd00:1122:3344:101::b]:63862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.042197967Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.042258998Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1859b2db-339d-4dd1-ac95-c9a068feee6b","remote_addr":"[fd00:1122:3344:101::b]:63862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.053003449Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"538d9154-b59a-4257-965a-1d1b6869cf91","remote_addr":"[fd00:1122:3344:101::b]:63862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.107846909Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.107964302Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"855c80a8-e611-46e8-97d0-5785890d0c18","remote_addr":"[fd00:1122:3344:101::a]:56174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.163261552Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.163292662Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f7adb191-c44c-433e-b607-f7bf7f384a33","remote_addr":"[fd00:1122:3344:101::a]:56174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.175179609Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3e267a1a-e5c5-4d16-875c-4d23ddc0bb60","remote_addr":"[fd00:1122:3344:101::a]:56174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.17524202Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56725"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.175350203Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.223081693Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.317816537Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"728affad-e92b-4100-83bd-255e7cf64db4","remote_addr":"[fd00:1122:3344:101::b]:56725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":142496,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.318071943Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.318336749Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"afa38a3e-0efc-4183-92b8-842f6b6219a9","remote_addr":"[fd00:1122:3344:101::b]:56725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.31840292Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"962a92de-f2bf-4268-a0d4-88fba939d1dd","remote_addr":"[fd00:1122:3344:101::b]:50599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.342939631Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40262"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.343063504Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.388982403Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.465764435Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"462bcff6-6765-4ba9-bbe8-09bd02ab974b","remote_addr":"[fd00:1122:3344:101::a]:40262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122754,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.46598024Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45846"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.466017581Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.466296887Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f48600ba-8fd0-45de-904a-a16bde1d7d32","remote_addr":"[fd00:1122:3344:101::c]:63273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.466489781Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f2b57eb4-9b0c-4f0f-b66b-230e3dcb27ce","remote_addr":"[fd00:1122:3344:101::a]:40262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":296,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.466511712Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"82092c2e-ee91-4f7a-b7e4-bf7bf20da5c6","remote_addr":"[fd00:1122:3344:101::a]:45846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":284,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.529914193Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.529944134Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4697a556-4d55-4951-a89d-048cc6a28d1b","remote_addr":"[fd00:1122:3344:101::c]:63273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.541290489Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a6b9fbfc-d53c-4053-ac9c-ce38c16a74d8","remote_addr":"[fd00:1122:3344:101::c]:63273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":306,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.782693542Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.782881356Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"27f27be4-b2c2-49b1-9d43-4d522d7f9968","remote_addr":"[fd00:1122:3344:101::c]:56718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.866858099Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62449"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.866953111Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.905553586Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.958362041Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d27d1e82-6ea5-4c87-9d78-3f176bb1af6b","remote_addr":"[fd00:1122:3344:101::c]:62449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91439,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:20.958651447Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"393ca092-cb45-4aec-96b1-2ff150f6fa9e","remote_addr":"[fd00:1122:3344:101::c]:62449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:26.551246261Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"93326f89-66c9-4a00-a561-6f59093fea5d","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7334,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:47.564976545Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:47.565090688Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"63634c30-1995-4d11-8511-721cfd05b92e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56952","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.791998273Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37857"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.792179267Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c46a5e27-cd38-445b-9ffc-614e924a653d","remote_addr":"[fd00:1122:3344:101::b]:37857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.851900406Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.851927567Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"b37a61e1-0fe9-45ad-8df6-d7420aa39c03","remote_addr":"[fd00:1122:3344:101::b]:37857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.860319435Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"70d04a0e-0c2a-4d12-8a2f-9b4c14c8b8e8","remote_addr":"[fd00:1122:3344:101::b]:37857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.896447965Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59156"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.896618798Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:49.954013325Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.029757074Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d6a447fd-3429-412c-b6a4-97eae2ed41ee","remote_addr":"[fd00:1122:3344:101::b]:59156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133224,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.029916718Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.030124902Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"592d7fa2-ff27-43d0-a170-22ec87a66b78","remote_addr":"[fd00:1122:3344:101::b]:57049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.030280536Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7a1178ae-158f-420d-9e7c-07b221888a9c","remote_addr":"[fd00:1122:3344:101::b]:59156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.119728001Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44717"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.119873385Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"24b5650e-64a9-4298-b00f-bb0bec32f6a7","remote_addr":"[fd00:1122:3344:101::a]:44717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.163036733Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.163052793Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"be2c0e2f-b905-4417-be2b-864ada0ac43e","remote_addr":"[fd00:1122:3344:101::a]:44717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.182610182Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50156"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.182807706Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"0cea55a6-1ac3-461e-8db7-b1a9cd98a97c","remote_addr":"[fd00:1122:3344:101::a]:50156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.191281166Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5c241b62-b7d2-4e9f-aa74-c9dd8f2cd4ba","remote_addr":"[fd00:1122:3344:101::a]:44717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.213266289Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52101"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.213363201Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.266559494Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.34530611Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"029674d9-0388-4074-9dfe-97cc58195142","remote_addr":"[fd00:1122:3344:101::a]:52101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131971,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.345638807Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9555fa99-c4d8-4206-80fa-02c960d7f9b6","remote_addr":"[fd00:1122:3344:101::a]:52101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.408611979Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.408754122Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"65539c7c-edc8-4165-a891-791b04c980aa","remote_addr":"[fd00:1122:3344:101::c]:56836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.455334827Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.455359388Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8b11501f-0bb8-48d8-8727-b3e55b27ac6f","remote_addr":"[fd00:1122:3344:101::c]:56836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.471920049Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.472103653Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"65e94ef5-636b-43aa-86c9-519b52ed75d4","remote_addr":"[fd00:1122:3344:101::c]:47481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.475157432Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"70c336a0-6bd3-428e-a9ce-87efe79d63a3","remote_addr":"[fd00:1122:3344:101::c]:56836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.486273881Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39886"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.486370193Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.526113184Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.587603303Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"38c427c3-06c5-4342-8ac7-4761de35e307","remote_addr":"[fd00:1122:3344:101::c]:39886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101267,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:50.58791533Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c1b9a643-ab7f-4612-b3df-197a8c63b8a3","remote_addr":"[fd00:1122:3344:101::c]:39886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.177987588Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.178175342Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"b860f18e-f800-4c1b-ab21-a1bdfccde516","remote_addr":"[fd00:1122:3344:101::a]:35877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.178884938Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:874","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(14521601) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.178912468Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"PUT","req_id":"7602aa1f-19ff-42d8-b575-500c9e746a32","remote_addr":"[fd00:1122:3344:101::a]:35877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":564,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.224195254Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48585"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.238473344Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 66e8bce1-b95a-4433-886f-e35fda6d9a14, project_id: 40304ee1-a5ea-4982-8a07-932d1d13f50e }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-12-20T21:33:53.224007949Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(14521601))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-a904c0b0204\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 38c4a6d3-4a66-4d59-9ce4-1a161311f8f2, read_only_parent: Some(Volume { block_size: 512, id: 2c7a445c-41d0-4b34-920f-760b7593257a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 91592e9a-b5a7-43bc-98a3-7bd55486dc21, key: Some(\"L1RBpjkH8o7Ze4zcDxmP4nc/iV3l5OQpC+DPWHt318Q=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 38c4a6d3-4a66-4d59-9ce4-1a161311f8f2, key: Some(\"Fu4eYTUM/DsQpPH90U9M2RxK02z0cEwimLPybWLPbxE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-a904c0b0204\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.238514155Z","hostname":"buskin","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.238519705Z","hostname":"buskin","pid":1566,"propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(14521601))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f63b0c86-4482-4df4-a8b3-bfaa6e884045, kind: Instance { id: 4e1d5d1c-803b-4f76-ae75-2577f45db478 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(14521601), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-a904c0b0204\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 38c4a6d3-4a66-4d59-9ce4-1a161311f8f2, read_only_parent: Some(Volume { block_size: 512, id: 2c7a445c-41d0-4b34-920f-760b7593257a, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 91592e9a-b5a7-43bc-98a3-7bd55486dc21, key: Some(\"L1RBpjkH8o7Ze4zcDxmP4nc/iV3l5OQpC+DPWHt318Q=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::12]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 38c4a6d3-4a66-4d59-9ce4-1a161311f8f2, key: Some(\"Fu4eYTUM/DsQpPH90U9M2RxK02z0cEwimLPybWLPbxE=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19000\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-a904c0b0204\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-12-20T21:33:53.224007949Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.24009338Z","hostname":"buskin","pid":1566,"uri":"/vmms/f6bad467-a635-4e8a-b9bc-5d0c877ff60c","method":"PUT","req_id":"7a179ed9-53a8-4581-a387-8e377a916802","remote_addr":"[fd00:1122:3344:101::a]:48585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":15837,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.363115409Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.364383807Z","hostname":"buskin","pid":1566,"uri":"/vmms/f6bad467-a635-4e8a-b9bc-5d0c877ff60c/state","method":"PUT","req_id":"2036ff89-7235-436b-9666-f1e73f80f7da","remote_addr":"[fd00:1122:3344:101::a]:39795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1158,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.366952475Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 14521601 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.367000646Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.404699511Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.428326891Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.537111681Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.537320505Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8eaf2d12-6870-46e7-a957-e2f31810c700","remote_addr":"[fd00:1122:3344:101::a]:41104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:53.558601592Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {f63b0c86-4482-4df4-a8b3-bfaa6e884045: {10.151.1.176: {499d5cba-788b-419e-a0b2-9441bf49dce5}, 10.151.1.175: {499d5cba-788b-419e-a0b2-9441bf49dce5}}} }"} {"msg":"Profile for oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:55.311068927Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c zone","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:55.311262342Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 43.235403ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:55.485331675Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 119.75076ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:55.538938437Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 144.93985ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:55.673088575Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 594.855684ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:55.834660618Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 630.724475ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:56.440489512Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:33:56.552487144Z","hostname":"buskin","pid":1566,"uri":"/8c74ff24-982c-40fd-86fd-ef04898c5922","method":"GET","req_id":"bbb5910f-9769-46ba-92a1-cd42212b74f6","remote_addr":"[fd00:1122:3344:101::d]:54456","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8512,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 720.556586ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:57.082663942Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 925.347008ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:57.814420671Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 540.218686ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:58.751452212Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 1.0720032s","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:33:59.303223134Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 962.54523ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:34:00.386779031Z","hostname":"buskin","pid":1566,"zone":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:01.377396594Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:1786"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 70.266652ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:34:01.38615085Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 102.557207ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:34:01.465751285Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\") failed: Property not found. retry in 255.531651ms","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:34:01.577862359Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:01.843535167Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:1795"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:01.884011253Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:1819"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.028650667Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.030177891Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.030249783Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"84858851-0f24-469d-bfb1-6459d19e28ed","remote_addr":"[fd00:1122:3344:101::a]:41104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8471740,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-12-20T21:34:02.030627511Z }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.030660132Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.030676372Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-12-20T21:34:02.030627511Z }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.030685722Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-12-20T21:34:02.030627511Z }, migration_in: None, migration_out: None }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-12-20T21:34:02.082535335Z }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.082561896Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.082582966Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-12-20T21:34:02.082535335Z }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.082602737Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-12-20T21:34:02.082535335Z }, migration_in: None, migration_out: None }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:02.107900714Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"34c41641-0f16-4ed3-85c1-a43877b06f45","remote_addr":"[fd00:1122:3344:101::a]:41104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":487,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.315189429Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54025"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.315770722Z","hostname":"buskin","pid":1566,"uri":"/vmms/f6bad467-a635-4e8a-b9bc-5d0c877ff60c/state","method":"PUT","req_id":"6529ca62-45b6-4ad1-938a-069bb1cf4faf","remote_addr":"[fd00:1122:3344:101::a]:54025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":498,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-12-20T21:34:11.315869054Z }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.315892685Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.315912255Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-12-20T21:34:11.315869054Z }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.315929026Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-12-20T21:34:11.315869054Z }, migration_in: None, migration_out: None }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-12-20T21:34:11.355338669Z }","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.35536039Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.355392181Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-12-20T21:34:11.355338669Z }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.355400071Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.355408351Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.355643656Z","hostname":"buskin","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:11.355700728Z","hostname":"buskin","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/648bb898-81e2-4bc5-92c0-fede81914617/debug/bundle/zone\", \"/pool/int/c4a1ae97-37ab-4b25-9caf-f353d764ffc2/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/9d337a02-8b44-4fe3-b02d-dffc8c87f032/crypt/debug\", \"/pool/ext/fd3cb939-7a27-4242-9684-f71fce5b8238/crypt/debug\", \"/pool/ext/b35cb2aa-7e4b-4820-82a1-1376e0cb1176/crypt/debug\", \"/pool/ext/7d8d774a-5a14-4ea7-9916-6f77f3cc41c0/crypt/debug\", \"/pool/ext/dfc55466-0ce1-427c-805f-4acd9ba37666/crypt/debug\", \"/pool/ext/e507db4a-2c57-4f91-87c2-2f1d428e74d8/crypt/debug\", \"/pool/ext/b7548ed7-f361-44b2-a96e-7c703af98968/crypt/debug\", \"/pool/ext/715f86b6-8047-4de8-af26-ede400e16cd7/crypt/debug\", \"/pool/ext/305d4c56-8d7f-48cd-9171-b2ebd9a8936c/crypt/debug\"] }","zone_name":"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:12.096330774Z","hostname":"buskin","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c\", bundle_id: 4b431e9b-430e-43a7-aaa9-e1eeeb2aff3e }, time_created: 2024-12-20T21:34:11.362669934Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_f6bad467-a635-4e8a-b9bc-5d0c877ff60c","v":0,"name":"SledAgent","level":40,"time":"2024-12-20T21:34:12.096399576Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:13.903404055Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:13.919542337Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(14521601), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:13.919579328Z","hostname":"buskin","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(14521601), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:13.920931938Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-12-20T21:34:13.920900177Z }, migration_in: None, migration_out: None }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:14.044163601Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-12-20T21:34:13.920900177Z }, migration_in: None, migration_out: None }","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:14.048744714Z","hostname":"buskin","pid":1566,"instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","propolis_id":"f6bad467-a635-4e8a-b9bc-5d0c877ff60c","instance_id":"4e1d5d1c-803b-4f76-ae75-2577f45db478","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.74634567Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46050"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.746508524Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0c0b42e9-da51-481b-a1c9-afde3826fa41","remote_addr":"[fd00:1122:3344:101::b]:46050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.761432009Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {f63b0c86-4482-4df4-a8b3-bfaa6e884045: {10.151.1.175: {499d5cba-788b-419e-a0b2-9441bf49dce5}, 10.151.1.176: {499d5cba-788b-419e-a0b2-9441bf49dce5}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.7614703Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c22173e0-445c-4aeb-805b-695d453d480d","remote_addr":"[fd00:1122:3344:101::b]:46050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.772285442Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b5b632bf-1703-4146-b588-e14ae187eb3a","remote_addr":"[fd00:1122:3344:101::b]:46050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.970030086Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:15.970253161Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4ebb5c9b-432a-4c5e-bd42-d8418fcb6b98","remote_addr":"[fd00:1122:3344:101::b]:51280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.0253744Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.025579134Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"6a7e3318-38e6-4ed9-ae81-307a4551c8bf","remote_addr":"[fd00:1122:3344:101::a]:55964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.204110688Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.20420786Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1f6092ce-086a-4765-8912-d3410ef05c37","remote_addr":"[fd00:1122:3344:101::a]:50154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.218788687Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.218862348Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.21891608Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"560f8b4c-3741-42e1-b7c5-2dc2a18a2896","remote_addr":"[fd00:1122:3344:101::a]:50154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.226693624Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a681d004-8222-44c1-bd2d-bffbd15b8f7a","remote_addr":"[fd00:1122:3344:101::a]:50154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.599564165Z","hostname":"buskin","pid":1566,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51029"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:17.599688928Z","hostname":"buskin","pid":1566,"uri":"/baseboard","method":"GET","req_id":"8753fc71-704b-437e-b8d3-6fa3ffe2f61e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51029","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:18.233310805Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:18.233409637Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f21fe85b-04e6-4f2e-b6f9-ee65c61c808c","remote_addr":"[fd00:1122:3344:101::a]:64886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:18.246967051Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:18.246988392Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"00227a51-1026-4d1b-b872-721a11d239c6","remote_addr":"[fd00:1122:3344:101::a]:64886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:18.255073403Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c37e16a3-9bbd-4790-bfb6-0e4d12866836","remote_addr":"[fd00:1122:3344:101::a]:64886","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:19.989471474Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52730"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:19.989667158Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a6ae5715-7681-4c2f-ad53-602d6b479a06","remote_addr":"[fd00:1122:3344:101::b]:52730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.058896291Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.058926531Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"975fd718-030b-451f-99b0-4900d11bb8d2","remote_addr":"[fd00:1122:3344:101::b]:52730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.068886175Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"59a5500c-289d-47f1-8f5c-7192555ddca7","remote_addr":"[fd00:1122:3344:101::b]:52730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":210,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.092408732Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36694"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.092512864Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.133127885Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.190893101Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fa7b08e7-c0be-453a-9e39-ecec19d5bada","remote_addr":"[fd00:1122:3344:101::b]:36694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98426,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.191091155Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41392"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.191203297Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1315be7a-52c1-4db0-889f-126b0c494134","remote_addr":"[fd00:1122:3344:101::b]:36694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.19132347Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"4737d0d2-5763-44e4-94d7-b0b6086110e6","remote_addr":"[fd00:1122:3344:101::b]:41392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.338319126Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46949"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.33847251Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e56f20be-a329-4a99-8ffd-bd90288ce961","remote_addr":"[fd00:1122:3344:101::a]:46949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.384653935Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.384738927Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1db661ee-ede3-45fc-baaa-a76e73c1ecaa","remote_addr":"[fd00:1122:3344:101::a]:46949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.394875975Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0f72b910-c1f6-4bf3-a0ff-fb7000b5fcbe","remote_addr":"[fd00:1122:3344:101::a]:46949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.40047789Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47828"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.400555912Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.455600116Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.530809893Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9251c009-3824-453f-bd7a-865346387c4e","remote_addr":"[fd00:1122:3344:101::a]:47828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130278,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.531027828Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54586"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.531097209Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.531275723Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"15d4a646-1b9e-4285-84b1-c1fe8da9d7ff","remote_addr":"[fd00:1122:3344:101::a]:47828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":176,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.531380325Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"00610a9f-4888-43c3-88f5-ac89524b668c","remote_addr":"[fd00:1122:3344:101::c]:54586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.531458897Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"2dc670c7-e292-47a1-a18b-ff925cf0723e","remote_addr":"[fd00:1122:3344:101::a]:57372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":235,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.579564056Z","hostname":"buskin","pid":1566,"sled_id":"8c74ff24-982c-40fd-86fd-ef04898c5922","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.579605377Z","hostname":"buskin","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d4ff9580-e313-4ad7-9e7f-1f13e77fe35e","remote_addr":"[fd00:1122:3344:101::c]:54586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.590321637Z","hostname":"buskin","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0ff5ac04-72eb-4082-9156-8fe9a3f6a94d","remote_addr":"[fd00:1122:3344:101::c]:54586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":223,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.629120857Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63325"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.629321272Z","hostname":"buskin","pid":1566,"uri":"/v2p","method":"GET","req_id":"09e8dd46-d517-4b4c-87f2-00854d177d50","remote_addr":"[fd00:1122:3344:101::c]:63325","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.728031295Z","hostname":"buskin","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58693"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.728126817Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.766690342Z","hostname":"buskin","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.8223557Z","hostname":"buskin","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6f8561c0-add6-43f4-a0ab-79d0f15b6c80","remote_addr":"[fd00:1122:3344:101::c]:58693","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94261,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-12-20T21:34:20.822652887Z","hostname":"buskin","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f8eecbd5-c90f-49b0-97e3-043be204bba9","remote_addr":"[fd00:1122:3344:101::c]:58693","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"}