[ Nov 9 02:43:14 Enabled. ] [ Nov 9 02:43:14 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Nov 9 02:43:14 Method "start" exited with status 0. ] [ Nov 9 02:43:14 Rereading configuration. ] [ Nov 9 02:43:14 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:14.870595737Z","hostname":"buskin","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:14.870788241Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:14.923723765Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:14.923761396Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.048067604Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.056950088Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.064839011Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.668037537Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.802167259Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.80221399Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.802265571Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.802347783Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.802372543Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.802386514Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805485207Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:15.805495548Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805510668Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805522128Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805548409Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.80559961Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805688502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805715572Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.805734313Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.815943034Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.815962604Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:15.94013999Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.505995454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.506039535Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.506045455Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.506057995Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.506067845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.506079286Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.506083526Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.514787166Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.514809116Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:16.624180997Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.17647979Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176527811Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176561672Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176568622Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176578572Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176586042Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176616583Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176635574Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176640483Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176645604Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176661634Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176666394Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176669764Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176673414Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176677414Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176686935Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176703315Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176707695Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176711555Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176715475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176719625Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176723725Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176740645Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176745886Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176749296Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176754206Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176771716Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176776836Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176780586Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176785486Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176789257Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176792557Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176810437Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176815007Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176844008Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176848778Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176919809Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176931709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.17693926Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176966Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176975041Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.176986401Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177012181Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177018621Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177023391Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177030702Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177035362Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177038672Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177047092Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177066702Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177070392Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177105513Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177109273Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177113753Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177117033Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177124504Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177149294Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177154094Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177178065Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177193535Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177204125Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177218935Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177223386Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177226946Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177259756Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.177268047Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.187681752Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:733"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.187706252Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999944509s","next_cleanup":"Instant { tv_sec: 1300, tv_nsec: 930578854 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.187746233Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.187784824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.187838685Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.189628542Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.190581722Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192253996Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192504191Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1083"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192510222Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192571093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192624024Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192634364Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192642694Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1146"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192668395Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192677115Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192684165Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192688345Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1188"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192692405Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192695715Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192721196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.192803077Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193327178Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 5bbfcf9f-0313-44b5-8699-fd964d8a4a29,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: f032ba4a-ac68-41c0-a068-00d3371df648 (sled),\n rack_id: 5bbfcf9f-0313-44b5-8699-fd964d8a4a29,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193591124Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193599954Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1222"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193661155Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193668875Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193828759Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.193957161Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.194018513Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f032ba4a-ac68-41c0-a068-00d3371df648 (sled), rack_id: 5bbfcf9f-0313-44b5-8699-fd964d8a4a29, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.194787669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4206"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.19485039Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4588"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.198997025Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.204352566Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.204855537Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209302938Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.20936057Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209482742Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:47175","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: f032ba4a-ac68-41c0-a068-00d3371df648 (sled), rack_id: 5bbfcf9f-0313-44b5-8699-fd964d8a4a29, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209609435Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209627175Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209660566Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209667166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209672616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209676106Z","hostname":"buskin","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209686017Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209701817Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209710977Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209714617Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209724507Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209730357Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209764958Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.209770398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.226876192Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.24757984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.265356287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.272062175Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:408"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.272095126Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.272100396Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:420"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.272134637Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 87.87542ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.275075898Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e285b0fc-d592-4a40-afe0-c3ae45c3ed21\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e285b0fc-d592-4a40-afe0-c3ae45c3ed21\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_290b5ea3-1da3-4d8a-a79d-8837f0b3c303/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.284689656Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_290b5ea3-1da3-4d8a-a79d-8837f0b3c303/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.309784465Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_290b5ea3-1da3-4d8a-a79d-8837f0b3c303/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.309823466Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.314990683Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:435"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.315005203Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_290b5ea3-1da3-4d8a-a79d-8837f0b3c303/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.315010383Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.363799822Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 102.989206ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.364348083Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"82d15d15-cb99-443f-bbca-a70e71a8098f\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"82d15d15-cb99-443f-bbca-a70e71a8098f\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.410993007Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.468343282Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 120.000248ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.468988255Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0e306097-f4e3-48ab-b6d7-09aac16aad1b\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0e306097-f4e3-48ab-b6d7-09aac16aad1b\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.589274061Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 170.990631ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.589727171Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c14da091-5e86-4af1-9eef-38f06cdc6302\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c14da091-5e86-4af1-9eef-38f06cdc6302\" }"} {"msg":"Mounting oxi_290b5ea3-1da3-4d8a-a79d-8837f0b3c303/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.630128106Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.643074773Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.643190865Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.643216526Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.643257927Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.762619364Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 203.421497ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.762846709Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"feb28244-1d5f-4eaf-a2b5-191b4c44c8fb\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"feb28244-1d5f-4eaf-a2b5-191b4c44c8fb\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.967685612Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 265.973084ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:17.96857462Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd9f9a33-eae6-4522-ac23-133efaf0956e\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd9f9a33-eae6-4522-ac23-133efaf0956e\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.235652749Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 432.577286ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.23617215Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"755c7e54-8a10-4542-a70f-a7279757e86b\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:43:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"755c7e54-8a10-4542-a70f-a7279757e86b\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.380785468Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.38664719Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:41739","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.386698971Z","hostname":"buskin","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.386735671Z","hostname":"buskin","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:41739","producer_id":"f032ba4a-ac68-41c0-a068-00d3371df648"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:18.387984427Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"205.29849ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.388903556Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.388921677Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.388978998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:917","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.388984898Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:18.59568051Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"422.682885ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:18.669639668Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:19.020270754Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"903.74405ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:19.925235897Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.846332651s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.793470679Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2664"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.79349688Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2882"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.7935024Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2534"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.79350699Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2581"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.7935113Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2963"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.79351534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3005"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.795524702Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2807"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.79690439Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2575"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.797027493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:20.797154565Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 63.355789ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:21.031371225Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 108.306078ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:21.228679983Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 281.170221ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:21.403796772Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 296.844229ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:21.776470234Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 825.589768ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:22.160814146Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:22.773363276Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.170033405s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 741.795898ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:23.105630652Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 672.204421ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:23.935776658Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.28173481s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:24.837460062Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.318072107s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:26.248970602Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:27.959060673Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.499638502s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/f032ba4a-ac68-41c0-a068-00d3371df648)","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:27.965608489Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.995266051Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.99569287Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.996700661Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.996719481Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.996730681Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.996741611Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.996905075Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.997073098Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.997114879Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.997124239Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3909"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:27.997175551Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:47175","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.189814292Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.189880153Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.190055087Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.190104418Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.190112848Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.190149999Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.19018708Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.19023947Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.190318522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.196189104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4230","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.196212274Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4246"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.196223554Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:28.411646646Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12239135335040497123"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:28.411779549Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13750704610523489124"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.411788059Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.440691617Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:44350"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.441084535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.441096245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.441159336Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.441175836Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.441287419Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"19ffdd79-3dc3-42e3-af1c-acf8cfe79b84","remote_addr":"[fd00:1122:3344:101::1]:44350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":387,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.471114155Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:36170"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.471261528Z","hostname":"buskin","pid":1572,"uri":"/sled-role","method":"GET","req_id":"ca8ee7b1-8396-4ae6-9dd4-204913139431","remote_addr":"[fd00:1122:3344:101::1]:36170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.471950112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.471967713Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.47230735Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.472573776Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:886"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 4d3e084b-214a-46ae-baed-e04e00554917, pool_id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 53c51f8b-256c-4157-b1c1-60bcbd6b8737, pool_id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: dc3854b2-62ab-4ec0-9afb-5993bd9aafde, pool_id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 76670b3b-776e-4857-af51-465a30d486d2, pool_id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 554f59ae-b23f-497d-8acf-32b9f1f65c3c, pool_id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c0696728-a1b0-4953-be39-29cad86f7b75, pool_id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: b7dec911-d4d9-4524-9ddd-751d9c2f2719, pool_id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 597cd284-a042-4532-9280-c5ceef96f9dc, pool_id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: fa782f29-5423-4acd-85d3-02fc6f0aa5e5, pool_id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.501680337Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:376"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.501778569Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:62394"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502172227Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:886"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 4d3e084b-214a-46ae-baed-e04e00554917, pool_id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 53c51f8b-256c-4157-b1c1-60bcbd6b8737, pool_id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: dc3854b2-62ab-4ec0-9afb-5993bd9aafde, pool_id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 76670b3b-776e-4857-af51-465a30d486d2, pool_id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 554f59ae-b23f-497d-8acf-32b9f1f65c3c, pool_id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c0696728-a1b0-4953-be39-29cad86f7b75, pool_id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: b7dec911-d4d9-4524-9ddd-751d9c2f2719, pool_id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 597cd284-a042-4532-9280-c5ceef96f9dc, pool_id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: fa782f29-5423-4acd-85d3-02fc6f0aa5e5, pool_id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502205738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502210618Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502223398Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502228308Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502254369Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502261899Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502269479Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.502274299Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.512278936Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.512315527Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"31d0192b-e979-447e-9118-b71dd165c143 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.628489318Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.636630586Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.636696317Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.636704757Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.636711368Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.636715988Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.63683455Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.63684056Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.677871258Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_31d0192b-e979-447e-9118-b71dd165c143/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.685669729Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_31d0192b-e979-447e-9118-b71dd165c143","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.887575622Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.895616368Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.895652269Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.895660799Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.89568615Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.89569423Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.905305469Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:28.905330079Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"99589249-e847-4d9d-a74c-b1f313d5d545 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.018673461Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.027287029Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.027312259Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.02732002Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.02733032Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.02734113Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.027457342Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.027467963Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.067537411Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f032ba4a-ac68-41c0-a068-00d3371df648)","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:29.071287368Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Automatically destroying dataset: oxp_99589249-e847-4d9d-a74c-b1f313d5d545/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.075750661Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_99589249-e847-4d9d-a74c-b1f313d5d545","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.282147686Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.290369896Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.290397217Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.290402046Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.290405577Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.290413037Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.300484845Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.300511266Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.411315356Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419003104Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419046955Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419052705Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419087446Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419108086Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419212149Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.419219999Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.461344409Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.469435887Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_74b698d6-a94b-4491-b5c6-a2bac3974e86","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.674695739Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.682952479Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.683016591Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.683027561Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.683037721Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.683047391Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.692948336Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.692976066Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"62a6cce5-1929-4693-88be-42b0a605236c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.805352409Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.812898765Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.812969046Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.812978877Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.812995517Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.813005747Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.81312268Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.8131313Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.856521886Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_62a6cce5-1929-4693-88be-42b0a605236c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:29.864400579Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_62a6cce5-1929-4693-88be-42b0a605236c","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.070455007Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f032ba4a-ac68-41c0-a068-00d3371df648)","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:30.074209684Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.078613566Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.078633416Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.078649696Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.078659307Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.078667957Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.088820857Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.088859257Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.200084316Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208343417Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208379757Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208387328Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208400038Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208411248Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208584892Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.208591832Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.248241981Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8b88214b-96a1-4c48-a234-7cd78b00e420/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.256103184Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_8b88214b-96a1-4c48-a234-7cd78b00e420","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.460264953Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.467999303Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.468037224Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.468076395Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.468082805Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.468088715Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.47850167Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.478548871Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"db69e33d-bffe-47ae-b048-940c028eae57 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.592965145Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600028291Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600060892Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600069182Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600081943Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600089513Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600208265Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.600216835Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.639745192Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_db69e33d-bffe-47ae-b048-940c028eae57/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.647717937Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_db69e33d-bffe-47ae-b048-940c028eae57","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.866370226Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.875019405Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.875087906Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.875099326Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.875107007Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.875113457Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.88445528Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.88448374Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:30.99821844Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.005992541Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.006030241Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.006036992Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.006068832Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.006139864Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.006217565Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.006236836Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.047054599Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_83c1e528-b687-4cd0-a6ee-e747f81e14fc/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.055624786Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f032ba4a-ac68-41c0-a068-00d3371df648)","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:31.076259323Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_83c1e528-b687-4cd0-a6ee-e747f81e14fc","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.259469889Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.267536376Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.267592727Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.267604538Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.267613598Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.267620198Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.276718916Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.276753796Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.386818791Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.393867617Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.393898708Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.393906768Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.393919558Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.393927368Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.39403866Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.394045451Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.43369151Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_736f1ad3-6d18-406b-b576-5193fe8077c2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.441311647Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_736f1ad3-6d18-406b-b576-5193fe8077c2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.639934972Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.646997348Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.647036819Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.647047109Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.64706812Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.64707586Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.657466684Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.657501155Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.767186182Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:181","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.775218458Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.775263799Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:227","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.775269809Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:231","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.77530164Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:233","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.77531232Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.775464943Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.775474054Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.822446084Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0c46804c-6ce7-415e-af40-2b1d35c7620b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:31.830701835Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288"} {"msg":"Looking for unencrypted datasets in oxp_0c46804c-6ce7-415e-af40-2b1d35c7620b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.042596143Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:374"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.051388535Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.051435956Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.051520427Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.05165099Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 290b5ea3-1da3-4d8a-a79d-8837f0b3c303 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd41f58e-e0a0-4e88-9b00-217e5d2f507b (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.051721912Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.051732492Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.051908126Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052209722Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:890"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052218672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052222752Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:916","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052227032Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:921"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052232082Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:926"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052246182Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052252653Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:932"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052286613Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052291713Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.052368855Z","hostname":"buskin","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"981c68fd-0b96-43ee-b4d3-db99a10a7471","remote_addr":"[fd00:1122:3344:101::1]:62394","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3550455,"response_code":"200"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f032ba4a-ac68-41c0-a068-00d3371df648)","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:32.07921515Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.084091581Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.084218673Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:45600"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.084513139Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.0845217Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.144256704Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.144306145Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.203881886Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.203928777Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.264871067Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.264904997Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.264920138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.264976419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.310341376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.310393088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.310403628Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.360654086Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.36419956Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.367753563Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.379556847Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.383955418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:32.387646364Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/f032ba4a-ac68-41c0-a068-00d3371df648)","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:33.081536714Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:33.965513043Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.442854947Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.44300637Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.443055961Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.443157143Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.929075976Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.929143407Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.932881284Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 46.49049ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:34.943121096Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:34.943164097Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 59.242745ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:34.998954759Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 35.331766ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.057275664Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 244.166787ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.065046215Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 48.810884ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.071610191Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 77.217754ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.100254883Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 137.462874ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.194079902Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 236.92931ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.245057885Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 482.911283ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.331190245Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 119.574902ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.352377823Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 237.029856ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.514643207Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 514.597305ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.529001594Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 624.957512ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.762752835Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 1.127283193s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:35.824659154Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 1.062648395s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:36.052764007Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 549.115986ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:36.397104244Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 1.246268052s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:36.955958363Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 1.237696253s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:36.965181734Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.089751978Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4254"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.089808479Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4264","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.096033958Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4343"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 520.29373ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:37.126028828Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.211274369Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4391"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.2113163Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4418"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.254601655Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4431"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.254652106Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4295"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.310691354Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4337"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.310730915Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4398","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.331104646Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4409"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.331136907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4451"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.409550357Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4483"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.409585578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4489"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.509893251Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4530"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.509943602Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.543311641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.555687657Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:41130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.555953353Z","hostname":"buskin","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"74aeae96-8a29-4b24-aab0-466d774ecbe5","remote_addr":"[fd00:1122:3344:101::2]:41130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.57566069Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.575767092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:731"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.57615307Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:734","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.57617252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.611757126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 1.19161177s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:37.655285186Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662360622Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662426113Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662437523Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662443754Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662450914Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662456574Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662462374Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662469474Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662475204Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662495455Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662503205Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662508985Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662514745Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662519905Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662523835Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662527835Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662531565Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662535145Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662538956Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662542796Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662546545Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662550436Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:37.662556886Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:37.967064409Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2477290899437388684"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:37.967188221Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"37899521630742692"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 1.132906672s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:38.211158363Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 726.103963ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:38.218528075Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:38.412867642Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12239135335040497123"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:38.413015825Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13750704610523489124"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 1.069341328s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:38.856744155Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 750.76208ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:38.953880983Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f\") failed: Property not found. retry in 678.356673ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:39.353566092Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da\") failed: Property not found. retry in 1.436351226s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:39.714031672Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b\") failed: Property not found. retry in 1.186629752s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:39.935126341Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.169468279Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_594379b7-4c26-4890-888e-966e710507da"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.16951245Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_bd994876-0077-4232-80e7-c0cfe919788f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.16952Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_internal_dns_504bb202-49f7-4613-ba58-ec4034140a6b"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.16953169Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.169890908Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.170164913Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"813bdb5b-57b0-4a13-ab53-7fd0c8f78e15","remote_addr":"[fd00:1122:3344:101::1]:45600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9085844,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.172575573Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.226579609Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.279562174Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:576"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.334397218Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.334408208Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.366704715Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.36741344Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.36743355Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.396668465Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.396761527Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53509"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.396982621Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.396991791Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.417130137Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.417180478Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.43516986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.435200511Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.453376397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.453432378Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.453609391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.453767055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.464574918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.497324885Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.49808807Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.498093961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.528086981Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 193, 255]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.528129122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.528498229Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.528509009Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.528751034Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.529172983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.553982556Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:41.571719492Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:42.191342307Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:42.19149441Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 64.292539ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:42.320627759Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 117.076097ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:42.402744376Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 169.372113ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:42.563235693Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 397.347561ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:42.743181462Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 750.757108ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:43.150269234Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 875.574216ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:43.910818602Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 1.298841442s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:44.796958005Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 698.604274ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:46.105536138Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10\") failed: Property not found. retry in 1.233779042s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:46.813022669Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:47.969496919Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2477290899437388684"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:47.969614581Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"37899521630742692"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.067136126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.067204338Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.067742279Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.068120207Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"5a48feaf-6d91-4313-b7d6-2da30953bac6","remote_addr":"[fd00:1122:3344:101::1]:53509","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6671286,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.070285242Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:653"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.100116638Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.100270241Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:37282"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:48.108381659Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.108512341Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"c08fd557-bf55-40b1-af14-89712fb62294","remote_addr":"[fd00:1122:3344:101::1]:37282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8024,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:48.110685036Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.204854192Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:48.205025116Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:57243"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:48.414617738Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12239135335040497123"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:48.41472783Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13750704610523489124"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:50.057258944Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5879590270478452129"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:51.144028343Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"18228604903066422143"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:51.171330077Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2483324700446410942"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3d812220-8c6c-4153-acc6-84bdcd8e5b10': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-09T02:43:52.21325453Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.213390673Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"a4e95258-f57f-47a3-b980-4831d8cc5e3a","remote_addr":"[fd00:1122:3344:101::1]:57243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008266,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:52.215562108Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.404767678Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.404948932Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:45531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.41360447Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"d07a7d8b-874c-4623-9eee-aa23cd90e3e1","remote_addr":"[fd00:1122:3344:101::1]:45531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8590,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.415067781Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:52.415073751Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.54662681Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.546791793Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:34370"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.555452052Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"4236ac77-03e7-46bb-b2b8-d9fcab83c21a","remote_addr":"[fd00:1122:3344:101::1]:34370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8588,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.558198249Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:52.558207569Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.820479149Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.820611132Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:35865"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.829530206Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"894cc93d-3c7a-4b60-913f-f68e982cbad2","remote_addr":"[fd00:1122:3344:101::1]:35865","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8860,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:52.830831883Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:52.830837873Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.107093872Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.107242695Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:54111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.115339543Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"82275731-533a-4561-bbf3-42bf9a62a75d","remote_addr":"[fd00:1122:3344:101::1]:54111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8046,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.118265973Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:53.118273363Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.396432552Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.396580255Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:53799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.406129992Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"da7029a6-613a-4caf-b8fa-264499ea9987","remote_addr":"[fd00:1122:3344:101::1]:53799","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9478,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.40747502Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:53.40748046Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.631367117Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.63153232Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:57396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.639688869Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"4e815bf0-26e7-4cb3-b06b-75e549fbc309","remote_addr":"[fd00:1122:3344:101::1]:57396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8097,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:53.641965876Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:53.641973126Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:683","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.070097859Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.070243992Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:60022"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.09192684Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"379e94ea-2a2d-4bb1-a24b-152129286a7f","remote_addr":"[fd00:1122:3344:101::1]:60022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":21569,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2907128903, ip_addr: 173.71.68.71, stratum: 2, ref_time: 1731120234.2058065, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.096632757Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:661"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.096642307Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1308"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.126677948Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.12676627Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:50998"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.126966344Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.126975774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.146383195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.146421996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.165444289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.16548135Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.18335919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.18340128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.243350809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.2433907Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.302764337Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.302810628Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.364477623Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.364523984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.425877602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.425916662Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.48726255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.487307091Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.487513855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.487674419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.571229126Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.571298227Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.571306507Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.571310987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.571316677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.663162686Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.667416923Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.671709912Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.676149044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.680680117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.685270692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.689968729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.69433266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.698865773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:54.703145292Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.351107114Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.351191916Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.357079898Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.357250061Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.363514001Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.363676844Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.368974954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.369155388Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.453923715Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:43:58.454067762Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 64.887247ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:58.494314592Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 71.135526ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:58.501673834Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 72.584688ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:58.508290081Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 69.273207ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:58.514834017Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 84.794777ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:59.477233669Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:59.517174588Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"245.912114ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 85.844556ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:59.517265049Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 114.019425ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:59.815189982Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 62.482556ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:43:59.896681291Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 262.572532ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:00.44917132Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 53.695141ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:00.813102161Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 237.900081ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.034621822Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 132.911981ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.162856879Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 245.460468ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.17834146Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 522.385721ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.191457092Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 134.364533ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.202336927Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 204.52428ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.283903618Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 423.513159ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.307148429Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 116.391734ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.347004285Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 520.3133ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.433354015Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 288.366183ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.472942785Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 1.033368516s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.498019375Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 665.187213ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.723257082Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 1.130506912s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.742231915Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 538.060461ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.770658034Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 1.005427651s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:01.963467419Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 1.33918409s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:02.317368853Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 1.037645258s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:02.396789929Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 575.753166ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:02.537929574Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 1.140466648s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:02.879148994Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 556.956418ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:02.974542741Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 633.680714ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:03.121697391Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 617.617596ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:03.440534599Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 694.971997ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:03.539662084Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 542.942171ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:03.662287594Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 1.329797876s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:03.76289177Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 1.123004341s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:04.026234818Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 587.222813ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:04.065622644Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 1.160520983s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:04.21353837Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 947.676725ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:04.241152782Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 744.406258ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:04.65893279Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 637.541974ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.097809276Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 879.534983ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.154579734Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 535.027357ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.196640654Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 896.640771ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.379487303Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 942.17233ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.40973623Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 978.859261ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.740478575Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 633.673794ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:05.748203635Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 640.036094ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:06.042215488Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 1.150055404s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:06.282956067Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 783.147264ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:06.358018693Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 750.801522ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:06.389599348Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 1.051627372s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:06.690486283Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 760.399468ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:06.725644251Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 724.699628ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:07.149563607Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 1.103896228s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:07.15840346Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 1.376736801s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:07.438911703Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 585.085006ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:07.492608316Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 827.968842ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:07.747337665Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 524.207528ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:07.881555267Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:07.958728276Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:07.95891376Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"b4b588ca-cada-4632-9810-6d1c3b58e87e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53643","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 985.419238ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:08.087339877Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 772.021599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:08.272030509Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 1.022980426s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:08.413879808Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 984.504597ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:08.582579635Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 659.479045ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:08.821406424Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 754.10237ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:09.051563133Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 1.378970829s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:09.079376079Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274\") failed: Property not found. retry in 1.24910245s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:09.443500616Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 977.069455ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:09.489706083Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 817.047214ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:09.574070582Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e\") failed: Property not found. retry in 1.005460712s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:09.812121154Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437\") failed: Property not found. retry in 1.26043859s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:10.398853013Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8\") failed: Property not found. retry in 1.429545015s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:10.465133776Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294\") failed: Property not found. retry in 527.024153ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:10.473765375Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.911965868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_6f93a92c-887b-4b99-aff5-d124dfb6a274"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.912017678Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_a32e4c9b-ed77-4c01-8805-a563861d6294"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.912026949Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.912033589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_b600967c-efa4-4e62-b453-f6649b578437"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.912038089Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_cockroachdb_336b54c4-a598-42fa-ab4f-faae3d67dff8"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.91209805Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.91257217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.912897307Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"16fd1e47-f9f8-404d-a112-4a78bfec1086","remote_addr":"[fd00:1122:3344:101::1]:50998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":17056239,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.945910381Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43997"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:11.973732747Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3705"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:15.957944298Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"397.462131ms"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:21.285038537Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3723"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.721221951Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.721327343Z","hostname":"buskin","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"f577cf6b-6888-486d-bd69-a1368c3736e9","remote_addr":"[fd00:1122:3344:101::1]:43997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":12809945,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.756355119Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.756513182Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:39995"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.757040632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.757047453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.782094672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.782136013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.805102489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.80513895Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.824862669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.82490057Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.84664663Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.846681081Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.865011671Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.865054012Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.88334227Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.883371911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.901278892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.901314073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.919929439Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.919958909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.992211057Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:24.992259188Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.069364456Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.069424307Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.154057831Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.154115092Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.22254319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.222601361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.338794659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.33884595Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.411953625Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.412004876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.489489552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.489538843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.565077148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.565144439Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.634085677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.634134858Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.702997175Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.703048426Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.766673115Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.766726616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.839569036Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.839619366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.839888062Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.840030655Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.94451338Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944563401Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944570351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944578132Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944586012Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944591722Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944598682Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944603982Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944611012Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944616762Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944640213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944646213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944654223Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944659773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944666403Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944671623Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944676774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944681934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944685374Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:25.944769745Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"bd1ad588-e60d-4baf-b092-82b1513d3940","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized)]","latency_us":1191221,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:25.944958049Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"123.069883ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd1ad588-e60d-4baf-b092-82b1513d3940\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd1ad588-e60d-4baf-b092-82b1513d3940\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.069123653Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.069457161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.069464098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.092331863Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.092380854Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.113953782Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.113999542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.133638569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.13368594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.154006161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.154048682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.17371497Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.173756921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.192159522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.192201013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.211621375Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.211672987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.23067841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.230719781Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.249146783Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.249188014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.269115797Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.269157988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.289076771Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.289123532Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.3102958Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.310350931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.332201984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.332246535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.35084003Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.350878611Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.372461419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.372503169Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.390761908Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.390800599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.40968667Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.409727321Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.427776835Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.427824886Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.447868911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.447909182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.467098049Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.467152361Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.467354535Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.467525978Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563659421Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563719022Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563734972Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563739452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563746682Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563752272Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563776123Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563808624Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563822894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563831934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563842614Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563853205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563861015Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563864895Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563869075Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563874495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563878135Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563894126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.563897826Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.563909236Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2944333f-95b4-486e-a5c6-9fd19dd40d04","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized)]","latency_us":495976,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:26.564078829Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"108.807608ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2944333f-95b4-486e-a5c6-9fd19dd40d04\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2944333f-95b4-486e-a5c6-9fd19dd40d04\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.673247131Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.673573318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.673597378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.696519253Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.696567694Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.717459717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.717505088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.73691659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.736963521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.753908312Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.753953583Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.772219602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.772258413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.79189035Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.79193434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.811076987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.811117908Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.828911467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.828950718Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.848541584Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.848582304Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.868088769Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.868131039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.888308877Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.888347888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.909513647Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.909561078Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.931921821Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.931964282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.952521138Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.952572559Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.973967852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.974015826Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.994999039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:26.995039999Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.016377451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.016416932Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.037028939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.03707209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.057664767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.057703438Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.07760242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.077648631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.077847725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.077963358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176120732Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176187993Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176196033Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176201314Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176207054Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176212784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176219884Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176225434Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176231034Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176247124Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176253264Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176259155Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176266105Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176271675Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176277205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176282325Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176291345Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176312756Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176326816Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.176460232Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9912bc91-426f-4209-8709-a9dbead04fae","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized)]","latency_us":504459,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.176622912Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"176.134522ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9912bc91-426f-4209-8709-a9dbead04fae\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9912bc91-426f-4209-8709-a9dbead04fae\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.353441706Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.353744423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.353752043Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.377325351Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.377370942Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.398733545Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.398789536Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.419127548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.419173349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.439782816Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.439825457Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.460095826Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.460139318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.480279445Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.480315325Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.499505713Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.499552604Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.520789364Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.520829835Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.544981265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.545029066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.566694455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.566744767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.58866118Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.588706801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.607754886Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.607807467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.629251491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.629294382Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.649498371Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.649541132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.669141398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.669180209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.688397767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.688447088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.707647126Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.707685587Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.727933446Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.727978817Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.747015782Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.747050763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.766071487Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.766126838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.766322912Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.766432604Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.863922524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.863986186Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.863994606Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.863999936Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864004966Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864010116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864016546Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864021626Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864026856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864031807Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864036847Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864041837Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864046767Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864050407Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864055547Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864059267Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864067717Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864074128Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864077637Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.864139619Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a77c89dc-d6e9-4ee3-a81d-445c6766c629","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized)]","latency_us":511962,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:27.864295872Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"126.84062ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a77c89dc-d6e9-4ee3-a81d-445c6766c629\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a77c89dc-d6e9-4ee3-a81d-445c6766c629\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.992411807Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.992694593Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:27.992702783Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.015378813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.015443434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.036149514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.036218935Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.059079503Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.05912675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.079979231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.080025337Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.1016397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.101682791Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.122757588Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.122796309Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.14262868Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.142668311Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.161605073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.161645104Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.180216019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.18026409Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.199383246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.199418677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.218747917Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.218787898Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.239587931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.23962991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.26054965Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.260619945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.280311843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.280355624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.301018552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.301064223Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.321238521Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.321282662Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.339931318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.339972819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.358088885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.358128006Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.376760062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.376801713Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.395577492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.395628043Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.395842927Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.395958019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497747719Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.49781133Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.49781946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497824921Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497830281Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497835871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497842791Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497846901Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497873779Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.49787796Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497884307Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497889613Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497896622Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497902592Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497909352Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497914703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497920272Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497926943Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.497930873Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.498034505Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"07ab0dd0-60bb-4ff6-9006-ae0669d74808","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized)]","latency_us":506883,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:28.498196028Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"168.464354ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07ab0dd0-60bb-4ff6-9006-ae0669d74808\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07ab0dd0-60bb-4ff6-9006-ae0669d74808\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.667185009Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.667477575Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.667488425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.688866328Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.68891891Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.707549396Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.707605637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.725724472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.725774383Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.74393773Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.743979801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.762474944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.762515375Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.781005428Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.781051099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.800145994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.800185545Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.818608197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.818650578Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.836957819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.837006638Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.855609574Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.855652455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.874045666Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.874091337Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.892788514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.892831613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.912197367Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.912240948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.930440625Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.930484626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.948758074Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.948802445Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.968101645Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.968151766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.989014668Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:28.989062129Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.007392059Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.00743081Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.025721999Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.02575963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.043691501Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.043743442Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.043938787Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.044049159Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141714003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141782514Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141790804Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141797494Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141802725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141806365Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141811245Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141816035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141820795Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141824205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141829185Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141834235Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141839085Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141844186Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141849315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141872256Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141876276Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141879646Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.141885146Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.141989358Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"d9534b9c-48a7-4cc0-a55e-1e904ed4e1fa","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized)]","latency_us":475956,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.142182273Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"154.400755ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9534b9c-48a7-4cc0-a55e-1e904ed4e1fa\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d9534b9c-48a7-4cc0-a55e-1e904ed4e1fa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.298143685Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.298517282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.298531053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.323863298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.323915039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.347389575Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.347438166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.370125796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.370175887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.3924867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.392539841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.415353963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.415404279Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.438601165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.438647486Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.461448049Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.46149381Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.483442434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.483487655Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.508461903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.508509784Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.532564322Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.532614693Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.558272165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.558315186Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.580891323Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.580940713Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.602193474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.602237405Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.625057258Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.625103959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.647727538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.647771342Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.671050261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.671100622Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.69173284Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.691782661Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.713225885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.713292557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.733295671Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.733343222Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.756277538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.756339949Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.756520732Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.756658165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860195571Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860269982Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860282293Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860292003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860301643Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860311263Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860320723Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860341174Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860349584Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860357444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860367344Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860377345Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860384575Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860390005Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860395405Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860403115Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860411125Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860417065Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860422846Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:29.860430346Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"841591b9-8d53-4702-a562-f21d74af6959","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized)]","latency_us":563683,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:29.860606129Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"252.576886ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"841591b9-8d53-4702-a562-f21d74af6959\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"841591b9-8d53-4702-a562-f21d74af6959\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.113606492Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.113899178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.113908178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.138331095Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.138387066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.160654047Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.160704628Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.184557603Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.184614954Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.207958858Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.208010039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.232907635Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.232966306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.257670028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.257727649Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.280591473Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.280643114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.302978467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.303018268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.326846631Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.326890562Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.34994627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.349992391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.374189022Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.374250273Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.399301263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.399353944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.42138441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.421424211Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.44258376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.442632387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.464637157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.464684968Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.486751435Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.486797886Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.509160419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.509226381Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.529398736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.5294537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.550385664Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.550433795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.570797906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.570860017Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.571060311Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.571172124Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681601772Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681656143Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681664033Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681667973Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681673103Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681678303Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681685104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681690144Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681695044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681700324Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681704044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681708994Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681712494Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681717134Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681730515Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681736735Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681741605Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681745095Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681748535Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:30.681788826Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"fb1c7c46-75ea-4916-ad01-f036ac8a2d7d","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized)]","latency_us":569608,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:30.681945911Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"519.789553ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fb1c7c46-75ea-4916-ad01-f036ac8a2d7d\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fb1c7c46-75ea-4916-ad01-f036ac8a2d7d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.201367753Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.201644269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.201651209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.225462812Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.225500295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.24560943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.245641721Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.2610626Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.272834523s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.264609934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.264633204Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.285774552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.285809843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.30548235Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.305514671Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.326553597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.326587068Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.347585403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.347627584Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.371220673Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.371266944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.391443852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.391486853Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.410319853Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.410353814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.432284378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.432322189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.452373514Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.452411785Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.472015441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.472048442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.490834602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.490871722Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.510438158Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.510477129Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.532595887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.532634528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.554310747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.554354548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.573273519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.57330767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.592536909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.592572019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.610874269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.610915509Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.611122844Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.611243756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721393149Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.72145253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.72146091Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721484871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721492111Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721497541Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721502891Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721513321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721519201Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721525062Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721530432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721535752Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721541122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721548402Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721554002Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721557992Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721563192Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721570012Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721575363Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:31.721696855Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8f7ebd5c-e950-4a84-8953-728ceac33abd","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized)]","latency_us":521626,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:31.721859209Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"613.23543ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f7ebd5c-e950-4a84-8953-728ceac33abd\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f7ebd5c-e950-4a84-8953-728ceac33abd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.334450143Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.334725919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.334735159Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.358105793Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.358146264Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.378746121Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.378780442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.400639055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.400678825Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.419352303Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.419385893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.438449978Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.438487839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.457500403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.457535254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.475410244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.475447815Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.494221064Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.494259575Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.512988752Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.513026787Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.532046498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.532098159Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.553699436Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.553741077Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.574144039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.57418358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.595117184Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.595160185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.614562007Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.614597688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.633129322Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.633178083Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.654356381Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.654395902Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.673956678Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.673989888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.695025254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.695059035Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.714260983Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.714302994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.731228905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.731278426Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.73147706Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.731604072Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834679278Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83473625Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83474429Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83475027Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83475731Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83476401Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83476936Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.83477468Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834779751Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834784871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834810721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834817351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834821191Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834826421Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834830261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834835082Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834840642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834844122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.834848892Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:32.834911583Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"90dab596-0976-4b67-952c-da3908eb5614","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized)]","latency_us":501707,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:32.835068757Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"727.030918ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"90dab596-0976-4b67-952c-da3908eb5614\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"90dab596-0976-4b67-952c-da3908eb5614\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.56103569Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.561305265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.561312346Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.58375032Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.583799921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.603176913Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.603222804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.62476939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.624826602Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.643374476Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.643415647Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.662940392Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.662983762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.682520967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.682567048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.703151845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.703193326Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.723325093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.723367054Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.745091874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.745139535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.766540788Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.766580369Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.787214727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.787252448Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.807237662Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.807283073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.827150754Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.827193305Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.850281624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.850322584Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.870917051Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.870959572Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.891167681Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.891204062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.911548283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.911585874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.931374634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.931415465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.950839238Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.950877318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.972153509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.97220685Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.972401614Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:33.972511787Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074553061Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074609672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074627533Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074633073Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074637873Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074642993Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074647973Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074653323Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074658233Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074663073Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074668244Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074673824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074677234Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074682234Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074685754Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074690824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074696184Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074699574Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.074716855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.074778987Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b2c493fd-da34-4796-ab1e-ab4e39b60ec0","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized)]","latency_us":515045,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:34.07494955Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"454.278454ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b2c493fd-da34-4796-ab1e-ab4e39b60ec0\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b2c493fd-da34-4796-ab1e-ab4e39b60ec0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.5290652Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.529364816Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.529372796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.553600738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.55365117Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.575767987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.575812828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.596023437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.596070988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.616768407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.616812938Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.638771633Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.638825214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.660044163Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.660089654Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.68156598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.68160936Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.703150537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.703194448Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.723473968Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.723515909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.745038185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.745082986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.765463638Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.765507079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.789463555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.789494366Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.812623615Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.812655866Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.833271773Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.833313374Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.854724328Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.854759509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.875123351Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.875160271Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.895697677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.895734518Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.915886205Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.915923566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.937067794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.937123225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.95857664Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.958619831Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.958798674Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:34.958918137Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065052266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065107428Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065113828Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065119208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065124168Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065130558Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065135578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065140958Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065146138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065158749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065163849Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065169059Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065174029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065177719Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065182809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065187659Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065192479Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065197149Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.06520199Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.065258091Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"6d183f16-1e5e-477f-a531-be972c460c1f","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized)]","latency_us":537533,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:35.065430964Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"783.657975ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6d183f16-1e5e-477f-a531-be972c460c1f\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6d183f16-1e5e-477f-a531-be972c460c1f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.847967193Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.848292206Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.848298986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.872158211Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.872208402Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.893424892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.893469893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.913389685Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.913426216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.933957772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.933993552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.953537097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.953575848Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.974217216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.974254567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.99228758Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:35.992331491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.011400956Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.011438557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.037755043Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.037793393Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.059277989Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.059316839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.078993087Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.079031698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.098602964Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.098639324Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.119638189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.11967637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.140862589Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.14090014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.164187392Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.164227223Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.185181598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.185218658Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.20555326Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.20559213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.22581731Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.22585595Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.245278553Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.245316834Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.265288218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.265337019Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.265543243Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.265662045Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366784841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366884293Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366894693Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366902893Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366910633Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366918974Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366925704Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366932774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366940754Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366948524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366954814Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366962514Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366969335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366977415Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366985655Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.366992205Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.367007275Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.367016036Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.367022266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:36.367130548Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"80d96c03-568f-4291-a6bb-0310b0d11516","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized)]","latency_us":520455,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:36.367291441Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"991.161261ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"80d96c03-568f-4291-a6bb-0310b0d11516\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"80d96c03-568f-4291-a6bb-0310b0d11516\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.357252735Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.357547582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.357560782Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.380203581Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.380251352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.39994072Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.399981981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.421012567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.421060368Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.440059302Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.440107543Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.458767829Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.45881541Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.478787424Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.478827975Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.495912369Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.49595688Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.515983165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.516027276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.535674303Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.535722294Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.555356865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.555399911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.576476168Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.576517868Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.598003864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.598047304Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.618758424Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.618800134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.639543924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.639600916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.658868635Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.658925006Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.678856839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.67889762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.697974508Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.698015156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.715896847Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.715940228Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.737569016Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.737608667Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.758081991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.758124932Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.758333286Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.758456539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858382169Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.85843245Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858440631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858446241Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858451831Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858457011Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858462321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858468941Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858493182Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858498866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858504557Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.85851171Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.85851756Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858523926Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858528942Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.85853378Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858538702Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858543543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858547083Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.858595254Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"72f8e8c8-64ec-4d4e-8f31-f491f99dc2a9","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized)]","latency_us":502599,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:37.858722657Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.293182726s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"72f8e8c8-64ec-4d4e-8f31-f491f99dc2a9\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"72f8e8c8-64ec-4d4e-8f31-f491f99dc2a9\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.912538742Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36440"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:37.912664234Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"1126bcd7-e8a8-4504-9b3c-4fecaae679be","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36440","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.149697878Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.149982244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.149990224Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.173211465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.173265157Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.196524159Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.1965689Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.21687465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.216927011Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.236416405Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.236457656Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.255495001Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.255538691Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.274384682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.274438513Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.294805025Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.294845696Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.314101865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.314139506Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.332852864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.332891565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.352430339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.352476599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.371913413Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.371956204Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.390265723Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.390302864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.409092794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.409127884Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.428561707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.428594438Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.449791277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.449832548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.469381383Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.469411244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.488903938Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.488932498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.507670427Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.507702397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.526068128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.526104649Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.54358592Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.543626301Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.543839855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.543950422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640433267Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640490078Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640498858Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640505278Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640511199Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640517149Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640521419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640525759Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640531419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640537029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640543039Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640550469Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.6405711Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.64057727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.64058337Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.64058745Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.64059158Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.64059571Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.64059973Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:39.640607281Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"b90b102f-f448-43f2-9f03-f1c75284fddb","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized)]","latency_us":492130,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:39.640775294Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.762356965s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b90b102f-f448-43f2-9f03-f1c75284fddb\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b90b102f-f448-43f2-9f03-f1c75284fddb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.399775484Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.40003822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.40004654Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.423641779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.42367248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.444355939Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.444381889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.462729949Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.4627541Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.482811555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.482843566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.50087745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.5009045Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.518487295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.518511115Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.53613774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.536163801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.555610353Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.555653774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.575559357Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.575581757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.595844617Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.595871627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.614635056Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.614659147Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.634926717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.634953087Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.656264449Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.65630042Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.677821716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.677858246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.696551384Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.696580525Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.715323083Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.715363974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.734158793Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.734187944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.754053976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.754078166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.772654471Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.772683422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.791553282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.791594023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.791802028Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.79191571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890567734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890630306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890638346Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890644076Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890659456Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890665796Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890673017Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890679037Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890684837Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890690517Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890697417Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890701287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890706697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890712037Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890717388Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890724358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890729698Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890735168Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.890740638Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:41.890875521Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"392d9b56-8d41-40df-8bb3-57c52f5dc43a","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized)]","latency_us":492340,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:41.891031884Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.465908386s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"392d9b56-8d41-40df-8bb3-57c52f5dc43a\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"392d9b56-8d41-40df-8bb3-57c52f5dc43a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.354145123Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.354652244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.354660184Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.377068428Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.377118999Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.397843829Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.39787934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.418000786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.418049407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.436566391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.436605472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.455359182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.455409932Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.474280212Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.474330134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.493391839Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.493427209Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.511710178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.511741949Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.533204323Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.533242004Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.552039733Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.552088303Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.57310122Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.573149221Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.59482447Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.594870881Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.614038358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.614075949Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.632368418Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.632414019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.651442803Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.651481484Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.67105841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.67109113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.690420231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.690456991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.709377154Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.709417304Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.729391898Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.729436769Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.748431053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.748483424Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.748685138Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.748815201Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845319851Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845360952Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845368462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845373982Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845379112Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845383872Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845388612Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845393752Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845398702Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845403522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845408743Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845423813Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845428793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845433813Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845438993Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845442383Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845447373Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845450974Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845454614Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:43.845534845Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4773aef6-5d18-40bb-b471-133e1ad4f0fd","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized)]","latency_us":492535,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:43.845695139Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.567829318s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4773aef6-5d18-40bb-b471-133e1ad4f0fd\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4773aef6-5d18-40bb-b471-133e1ad4f0fd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.410828982Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.411138948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.411146538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.43440021Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.434433031Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.454007156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.454037367Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.4739524Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.473998111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.491651456Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.491675477Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.511681362Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.511706732Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.52949368Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.5295149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.550965255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.550990956Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.570384827Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.570414438Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.589904422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.589927403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.60812492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.60815336Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.627685275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.627709666Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.646815901Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.646853382Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.66458536Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.66461423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.682227165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.682245845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.701287146Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.701310691Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.720327495Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.720349955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.738057712Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.738084933Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.755973553Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.755994994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.773391564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.773415145Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.791686113Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.791721954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.791939569Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.792082472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886231573Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886275904Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886285274Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886313985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886322285Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886328525Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886334965Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886343025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886348985Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886355025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886360906Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886368586Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886374666Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886382486Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886388966Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886395366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886402966Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886409036Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886423117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:45.886506129Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"f6e5608d-0570-4d93-ae61-e8a9ec2e2b9a","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized)]","latency_us":476844,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:45.886645881Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"1.696590692s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f6e5608d-0570-4d93-ae61-e8a9ec2e2b9a\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f6e5608d-0570-4d93-ae61-e8a9ec2e2b9a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:47.543535255Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"2.878526013s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.579595793Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.579862888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.579874768Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.60213646Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.60216614Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.622390709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.62241801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.643529547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.643574928Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.663236726Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.663273516Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.683075337Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.683113427Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.702261834Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.702302095Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.721399111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.721435792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.740143889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.74018097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.760001091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.760046522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.778953734Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.778987614Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.798599077Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.798634412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.818040474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.818085104Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.837822744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.837866845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.856533771Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.856581642Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.876041296Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.876083777Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.894875796Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.894910256Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.914403111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.914445872Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.933054837Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.933101228Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.951672173Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.951712064Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.970767059Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.97082804Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.971026724Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:47.971168887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068754609Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.0688322Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068849834Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068856818Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068863559Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068868003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068881472Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068889792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068909872Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068916142Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068922353Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068927373Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068932443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068938913Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068945933Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068957653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068963303Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068968123Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.068974614Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:48.068979714Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"85320c17-f48e-4b68-9bcb-afa751e026c5","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized)]","latency_us":490566,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:48.069149567Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"3.083901014s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85320c17-f48e-4b68-9bcb-afa751e026c5\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85320c17-f48e-4b68-9bcb-afa751e026c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.146563997Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.147009107Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.147025617Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.171963024Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.172009555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.19206262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.192118701Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.211314555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.211380151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.230804763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.230848764Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.252415491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.254847931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.271873014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.271920615Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.29144938Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.291492021Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.310105286Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.310145877Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.330247394Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.330299705Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.350698387Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.350745148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.37061673Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.370658811Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.390099604Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.390151215Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.410268502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.410317333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.429284666Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.429337317Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.448721229Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.44876297Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.468368016Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.468431827Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.487524533Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.489935403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.506825433Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.506880214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.52598423Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.526033591Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.545001464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.545078025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.545260869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.545373761Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649702003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649773045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649789515Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649807796Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649813656Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649818006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649822066Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649827266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649831216Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649836556Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649842266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649846246Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649850116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649858286Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649863717Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649884427Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649893387Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649897407Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.649901627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:51.649907207Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"e39a70ea-776a-40e7-990c-c6de392836b8","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized)]","latency_us":504517,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:51.65003702Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"4.385434655s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e39a70ea-776a-40e7-990c-c6de392836b8\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e39a70ea-776a-40e7-990c-c6de392836b8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.02555175Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.025882897Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.025891137Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.049395374Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.049452365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.0709178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.070962251Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.092509257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.092553258Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.111274846Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.111327027Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.130885013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.130935434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.148422876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.148469887Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.168021422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.168064893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.187374963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.187427134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.206876397Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.206933038Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.224696027Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.224739932Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.24320029Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.243244481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.264135054Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.264198885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.284074837Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.284120878Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.30302922Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.303077261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.32135605Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.321402091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.339556727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.339596358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.358710944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.358753125Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.37686917Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.376925891Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.395706361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.395761391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.416568133Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.416633044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.416861779Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.416989802Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.51104392Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511120351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511145382Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511171422Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511181742Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511207993Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511216443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511233863Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511248014Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511253844Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511259644Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511267094Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511274915Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511286915Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511292635Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511300595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511308345Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511315695Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.511327236Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:56.511332856Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"2a9b78ea-cafd-4bdc-b04d-381c1d7915ca","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized)]","latency_us":486940,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:56.51153255Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"2.850692242s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a9b78ea-cafd-4bdc-b04d-381c1d7915ca\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a9b78ea-cafd-4bdc-b04d-381c1d7915ca\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.356141116Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.356596576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.356615156Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.380901809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.38094591Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.400771341Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.400829512Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.418949718Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.418999879Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.440169988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.440216959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.459773184Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.459823255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.479204657Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.479251698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.497084427Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.497138698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.515030948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.5150812Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.534444761Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.534494652Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.552653198Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.552698549Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.570532118Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.572843046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.589849109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.58990314Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.609053497Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.609100258Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.628673333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.628713714Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.646589995Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.646651046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.665552508Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.665594828Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.683739444Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.683780765Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.701655426Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.701712377Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.72117547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.721221961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.739282306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.739342447Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.73950753Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.739620322Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.83503798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835096561Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835104531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835112981Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835125542Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835132492Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835138992Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835145922Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835152692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835159612Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835164273Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835170682Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835175813Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835182153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835196153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835212783Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835224714Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835234614Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835239304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:44:59.835244164Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"1607c09a-63a9-44fc-b1ba-9e3c6130c4f7","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized)]","latency_us":480216,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:44:59.835392857Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"6.622915412s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1607c09a-63a9-44fc-b1ba-9e3c6130c4f7\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:44:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1607c09a-63a9-44fc-b1ba-9e3c6130c4f7\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:05.409580136Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"4.364195962s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.441651002Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.442085149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.442097331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.464111358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.464164679Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.484731854Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.484777805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.505297261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.507718931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.523523948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.523572859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.54436606Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.544415891Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.565214692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.565285314Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.584235896Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.584278197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.603116808Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.603162229Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.621467918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.621523539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.641247978Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.641293589Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.660303743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.660345844Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.678130942Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.678182003Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.697066875Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.697120556Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.71520012Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.715245131Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.733593802Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.733643552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.752568695Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.752606176Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.772448447Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.772499478Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.790392819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.790441049Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.809064389Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.809099678Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.826923125Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.826977017Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.827172261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.827296703Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921755421Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921799552Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921806452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921821442Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921843183Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921857433Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921864763Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921869203Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921874773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921878743Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921884393Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921890124Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921901984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921907514Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921911594Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921919394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921924784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921935365Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.921941045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:06.921959325Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"ebad297f-740d-4661-bf31-7a4aef656425","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized)]","latency_us":481473,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:06.922135139Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"4.449176033s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ebad297f-740d-4661-bf31-7a4aef656425\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:45:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ebad297f-740d-4661-bf31-7a4aef656425\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:07.863325512Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:07.863465755Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3aa4cf2e-7ba4-4c13-86d1-e59fc9f58666","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34648","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.360285126Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.360740346Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.360749581Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.382721481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.382786412Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.402484651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.402538112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.42177123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.421818441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.442252675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.442299636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.45990414Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.459954261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.477730809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.480094128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.49418053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.494227631Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.512807316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.512845167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.53372893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.53617788Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.55354708Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.553606171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.572636836Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.572680417Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.595301175Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.595357787Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.615022954Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.615080105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.634205622Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.634255473Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.653130744Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.653169525Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.672807152Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.672864553Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.692106321Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.692158453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.708460251Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.708518132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.727006645Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.728888314Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.744192451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.744240532Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.744421086Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.744548448Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.84015348Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840231261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840241631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840249831Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840258792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840270252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840280122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840308913Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840319833Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840329153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840336723Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840346073Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840355644Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840378084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840384374Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840392024Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840400994Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840408605Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840416515Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:11.840488356Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"6a114278-345d-4920-8b2b-fb049ca77df8","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized)]","latency_us":481330,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:11.840768722Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"3.505261595s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a114278-345d-4920-8b2b-fb049ca77df8\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:45:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a114278-345d-4920-8b2b-fb049ca77df8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.338080304Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.338555214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.338572214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.362964109Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.363029411Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.383143548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.383195489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.401653251Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.401701462Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.423126036Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.423170617Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.440916505Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.440972106Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.460456439Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.46050307Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.478402541Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.478449992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.4976799Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.497734792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.516346437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.516389298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.538758032Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.538801643Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.558794637Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.558847028Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.577662408Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.580118099Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.595764383Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.595803104Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.614343888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.616305179Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.632334571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.632388262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.650424266Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.652789275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.669848498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.669904529Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.688467864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.690857863Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.708210443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.708254174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.727081624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.727131145Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.72736883Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.727508013Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822840459Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.82291521Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822936391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822946441Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822968971Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822976852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822986622Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.822996582Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823013492Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823032493Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823040923Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823059393Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823069173Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823079283Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823087744Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823111944Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823121835Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823127764Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823134465Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:15.823165845Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"cef930c4-2926-4746-b686-98b06b54e050","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized)]","latency_us":486278,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:15.823322339Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"6.969288185s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cef930c4-2926-4746-b686-98b06b54e050\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:45:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cef930c4-2926-4746-b686-98b06b54e050\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.774856418Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.775110003Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.775118463Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.796801263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.796858384Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.815222355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.815263275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.833582565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.833645436Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.853951727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.854006728Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.873128365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.873168245Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.893101419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.89315148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.914237206Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.914284187Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.936487787Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.936542519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.957250988Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.957299339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.976479726Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.976527747Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.996603233Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:22.996646474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.016648659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.01669981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.036804376Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.036857738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.055421432Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.055463973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.07704395Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.077094781Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.096657017Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.096705308Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.11802642Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.11807263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.138764069Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.138825475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.158166251Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.158212772Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.176797627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.176855059Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.177045803Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.177151495Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280031367Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280100868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280113888Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280120339Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280125889Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280137599Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280153499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.28016678Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.28017219Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.2801776Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.2801881Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.28019222Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.28019838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.28020397Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.28020948Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280230241Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280235961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280240311Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280245641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3277","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:23.280263511Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9faecabb-8048-4786-9a36-aeb9cc25648b","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\", TimeNotSynchronized), (\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\", TimeNotSynchronized), (\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\", TimeNotSynchronized), (\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\", TimeNotSynchronized), (\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\", TimeNotSynchronized), (\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\", TimeNotSynchronized), (\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\", TimeNotSynchronized), (\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\", TimeNotSynchronized), (\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\", TimeNotSynchronized), (\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\", TimeNotSynchronized), (\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\", TimeNotSynchronized), (\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\", TimeNotSynchronized), (\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\", TimeNotSynchronized), (\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\", TimeNotSynchronized), (\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\", TimeNotSynchronized), (\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\", TimeNotSynchronized), (\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\", TimeNotSynchronized), (\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\", TimeNotSynchronized)]","latency_us":506698,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:23.280445675Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:496","retry_after":"14.309385269s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9faecabb-8048-4786-9a36-aeb9cc25648b\", \"content-length\": \"124\", \"date\": \"Sat, 09 Nov 2024 02:45:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9faecabb-8048-4786-9a36-aeb9cc25648b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:24.757758878Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"8.800738906s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 594379b7-4c26-4890-888e-966e710507da (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 504bb202-49f7-4613-ba58-ec4034140a6b (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: bd994876-0077-4232-80e7-c0cfe919788f (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: af30a046-65b9-4298-a471-8b2ad03fd28a, kind: Service { id: 3d812220-8c6c-4153-acc6-84bdcd8e5b10 }, name: Name(\"ntp-3d812220-8c6c-4153-acc6-84bdcd8e5b10\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 255])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b600967c-efa4-4e62-b453-f6649b578437 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6f93a92c-887b-4b99-aff5-d124dfb6a274 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 336b54c4-a598-42fa-ab4f-faae3d67dff8 (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: a32e4c9b-ed77-4c01-8805-a563861d6294 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 0339f029-6c8d-4b23-83cc-6885965f777d (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be0f5b06-b4d4-43f1-a81e-d1635992ea60 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 84e31c38-00f6-4075-a1bb-afeeca692948 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 598f0700-44ae-48e4-9f3a-a3bd8eeee121 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: bb25f5f0-a387-4783-a156-b69bcd7dd4e4 (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: ff177c58-abbb-4bc0-bb54-67a185638e45 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0 (service), filesystem_pool: Some(ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4 (service), filesystem_pool: Some(ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0103f3de-a918-4681-9bb2-af8918749a48 (service), filesystem_pool: Some(ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6c9bae27-e783-49ed-b29e-bd849c13662c (service), filesystem_pool: Some(ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External } } } }, OmicronZoneConfig { id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 (service), filesystem_pool: Some(ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b (service), filesystem_pool: Some(ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc (service), filesystem_pool: Some(ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External } } } }, OmicronZoneConfig { id: 435c2cd2-2bd6-43cc-891d-385af930c2bb (service), filesystem_pool: Some(ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 71af5a4c-a704-4cf4-9d94-94819523084d (service), filesystem_pool: Some(ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.552280215Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:459"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.552559751Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.552573641Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 594379b7-4c26-4890-888e-966e710507da, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.577229952Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.579699634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 504bb202-49f7-4613-ba58-ec4034140a6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.597505483Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.597547233Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: bd994876-0077-4232-80e7-c0cfe919788f, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.617135059Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.617193681Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b600967c-efa4-4e62-b453-f6649b578437, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.6359985Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.636042041Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6f93a92c-887b-4b99-aff5-d124dfb6a274, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.655171068Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.655212539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: f5a578f3-a9fa-4dc2-a2fe-920d6aaa7d8e, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.673309563Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.673362515Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 336b54c4-a598-42fa-ab4f-faae3d67dff8, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.691728455Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.691774876Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a32e4c9b-ed77-4c01-8805-a563861d6294, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.710266699Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.71031019Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: e2d2fc82-684b-4404-ae54-ce48e33d41e3, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.729766093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.729820685Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.748327088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.748377329Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 84e31c38-00f6-4075-a1bb-afeeca692948, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.767128068Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.767181269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: c6a4db90-5649-4dbd-b94c-e7b51b493ee0, dataset_name: DatasetName { pool_name: ZpoolName { id: 31d0192b-e979-447e-9118-b71dd165c143 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.785942648Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.785986618Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 9ff0f763-588f-4b83-b5e3-b5b856e94da4, dataset_name: DatasetName { pool_name: ZpoolName { id: 99589249-e847-4d9d-a74c-b1f313d5d545 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.804103781Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.804151045Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0103f3de-a918-4681-9bb2-af8918749a48, dataset_name: DatasetName { pool_name: ZpoolName { id: 74b698d6-a94b-4491-b5c6-a2bac3974e86 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.818648715Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49180"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.818824299Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"88745bba-2394-43dc-91c4-5edb3c753159","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49180","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.823177569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.823194569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 6c9bae27-e783-49ed-b29e-bd849c13662c, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a6cce5-1929-4693-88be-42b0a605236c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.842087091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.842130462Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 09f2b2d1-ef56-4155-adc5-7f7fd3fc7188, dataset_name: DatasetName { pool_name: ZpoolName { id: 8b88214b-96a1-4c48-a234-7cd78b00e420 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.860577814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.860626655Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ca2a363-9d36-44f3-8de4-33bd220b1a2b, dataset_name: DatasetName { pool_name: ZpoolName { id: db69e33d-bffe-47ae-b048-940c028eae57 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.879066487Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.879119808Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3a74ab15-bc45-488a-93c4-dcadf3fc9fcc, dataset_name: DatasetName { pool_name: ZpoolName { id: 83c1e528-b687-4cd0-a6ee-e747f81e14fc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.897925158Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.897975889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 435c2cd2-2bd6-43cc-891d-385af930c2bb, dataset_name: DatasetName { pool_name: ZpoolName { id: 736f1ad3-6d18-406b-b576-5193fe8077c2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.916602555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.916650046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 71af5a4c-a704-4cf4-9d94-94819523084d, dataset_name: DatasetName { pool_name: ZpoolName { id: 0c46804c-6ce7-415e-af40-2b1d35c7620b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.935329073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.935387705Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.93566818Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:37.935798813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.031522946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.033016718Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115732792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115807183Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115829734Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115839954Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115849464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115867805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115899375Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.115920926Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.308161929Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.308381874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.308407855Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.308429375Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.308454636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.331927572Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.332195428Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.333539535Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.334920994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.335514196Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.340851617Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.345685547Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.350780453Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.355747705Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.360875432Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.366051239Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.371509522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.495892359Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.501056646Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.506160132Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.510804578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.515346712Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.520023459Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.524494462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.52971009Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.534783065Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.540137484Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.545445806Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.55095006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.556952774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.56298397Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.568990524Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.576411098Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.582560325Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.587469977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.594436871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.601434116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.672198213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.78345081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.788942962Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.788978023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.841867449Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 146, 126]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.841894829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:38.991571211Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.078065143Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.137398773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.13949622Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.139509224Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.18986809Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 137, 51]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.189901631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.191556785Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.191573645Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.244258577Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 185, 251]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.244295358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.246060235Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.246079535Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.304831623Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 189, 54]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.304867283Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.306559378Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.306585939Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.359818922Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 179, 126]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.359850613Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1260","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.360150089Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.360162109Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.361509117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363350765Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363362965Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363432117Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363441297Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363461407Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363467988Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363510079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363517519Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.363642941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.420580821Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.424659335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.428669548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.428931034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.428947804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.431852374Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.434698278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:39.434711122Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.20278358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.212578898Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.225564141Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.23944923Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.250632111Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.259681079Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.268098243Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.276937787Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.28528656Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:40.293163523Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:48.578250009Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"11.073940883s"} {"msg":"Profile for oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.727666366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.771587249Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.773553182Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.773568905Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.87019924Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.880833119Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.890368553Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.890948277Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.902410586Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.902955003Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.903137434Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.903442206Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.913879803Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.923734476Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.93313965Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.933344174Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.951395954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.951789018Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.970782453Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:48.970988098Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:49.092996844Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:49.093338572Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:50.115148424Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:50.115516162Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 57.820396ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.527180395Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 47.48941ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.538421237Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 63.307276ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.549716461Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 57.319089ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.56081308Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 42.775286ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.573618448Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 40.375119ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.604746898Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 69.603495ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:50.623212444Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 53.266895ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:51.904611909Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 66.143575ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.041261182Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 67.786361ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.143120177Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 86.816561ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.160154246Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 124.139556ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.168726425Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 127.240643ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.179530038Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 74.464872ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.193132549Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 149.21379ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.202789679Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 125.249736ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.216078333Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 128.861868ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.230310877Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 147.355765ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.241025898Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 108.630597ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.255976121Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 112.922666ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.292869541Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 43.647372ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.385931551Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 50.164917ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:52.583118974Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 170.213128ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.405424612Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 148.597014ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.451826418Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 137.850232ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.465906508Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 101.659933ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.498853289Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 285.464494ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.511318327Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 286.330375ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.522814858Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 236.582924ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.532158677Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 236.163708ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.541847057Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 264.166589ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.550117118Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 149.199261ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.559176095Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 128.713883ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.567367204Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 123.401607ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.576684027Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 442.406539ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.586351173Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 513.824627ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.612601409Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 211.868243ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.625079317Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 299.395007ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.636682835Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 114.76737ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.709039321Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 599.866695ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.722082092Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 300.258223ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.731929174Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 575.431752ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.799665343Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 310.082847ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.811156761Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 460.919963ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.822312659Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 264.149357ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.834539697Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 201.566775ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.847241396Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 364.580493ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.858129109Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 444.116451ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.868422911Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 868.040581ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:57.951845252Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 823.882087ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.043222971Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 585.342227ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.053355154Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.068886816s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.066542416Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.110974851s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.114054748Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 544.900609ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.13468126Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 459.680493ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.146438465Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.105540672s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.238946768Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 578.757085ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.298311762Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.047704946s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.326323478Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 687.049812ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.34069546Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 400.467238ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.389942245Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.169771349s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.619495169Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 571.020826ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.651474892Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.05578697s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.693007409Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 786.414852ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.804748513Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.261162856s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.833854516Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 562.254961ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.881779406Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.348806346s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:58.894615842Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 702.419803ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.04313206Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.181538234s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.178807143Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.338396553s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.275859049Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 574.909848ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.306212806Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.345201409s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.371069996Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 796.371025ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.387153128Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.416740074s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.458933871Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:59.536472604Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:59.53677738Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:59.536922803Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:59.54547806Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.468894758s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:45:59.606523012Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:59.607855859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:45:59.608249587Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 651.77494ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.322480917Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 566.703675ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.332150356Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 651.527343ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.341192193Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.416233956s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.350498516Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.239921278s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.360195207Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 970.562817ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.371934038Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:00.37298568Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:00.374781217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 897.518236ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.418171273Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.311113715s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.429723332Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:00.429818594Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:00.435840049Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 712.956877ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.627594921Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 47.385996ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.679413902Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 62.866444ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.689252475Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 33.411694ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.699341384Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 577.153055ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.730044486Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 36.974314ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.738992323Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 96.696113ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.748923308Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 115.334628ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.75822036Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 129.316114ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.767705596Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 26.135136ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.777470958Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 134.523792ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.812551983Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 80.712764ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:00.9449819Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 237.796603ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.034395098Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 224.647941ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.053003942Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 296.610822ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.076823834Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 922.550299ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.101567765Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.403002294s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.136471817Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 830.174976ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.148190219Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 965.909857ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.159256898Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.220146629s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.170398258Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 280.201233ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.186369788Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 201.126328ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.196002097Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 471.024891ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.287292584Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 279.071653ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.301771613Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.150440383s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.322326738Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.185234771s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.333317825Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 738.187224ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.354678686Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 545.600315ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.367925059Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 395.991194ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.386727998Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 307.763572ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.409817785Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 433.212753ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.480347182Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.190313329s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.594862022Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:01.598376712Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:01.598613556Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.02498556s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.617198632Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 756.850362ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.730965891Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.133919352s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.75269175Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 555.424666ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.770255593Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 976.577569ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.781808482Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.178978318s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.795076206Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 46.507071ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.817730074Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 96.851407ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.875134469Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 695.22463ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.927093614Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 662.209484ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.938395818Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 225.091732ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.98206268Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.092105605s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:01.992851103Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 678.052669ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.036079037Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 555.52521ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.105307817Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.00971648s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.138373191Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 285.048443ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.221048789Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 708.275789ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.339422065Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 701.67303ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.404485259Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 938.685082ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.486087585Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 1.39798453s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.501846711Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 801.562317ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.518989725Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 507.740521ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.531508684Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 728.971551ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.551871384Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 556.66362ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.613997808Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 508.961729ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.634781978Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 871.44907ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.65568283Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.047470902s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.673441857Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 537.287263ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.724580594Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 784.206245ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.771208517Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 541.548481ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.797509871Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 747.355738ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.901380637Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 754.282245ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:02.988737782Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 986.657567ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.052814467Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.294990802s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.066190313Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 939.876393ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.098759613Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.243966609s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.118516374Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.046831947s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.156291825Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.49241593s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.168433886Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.335760939s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.184919126Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 809.684004ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.27654213Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 620.339395ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.292953269Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 650.236801ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.332923015Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.34540262s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.350432087Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.467242634s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.437834552Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 772.68149ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.541926543Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 503.660847ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.569809517Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 761.96982ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.664381084Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.098968749s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.734440001Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.473107758s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.757056639Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 621.523266ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.915682747Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 954.909607ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.927637254Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 725.005939ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:03.996513347Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 796.774325ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.053821121Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 902.41271ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.067997684Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.117570487s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.08713942Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 807.186314ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.100644649Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.014640546s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.216249828Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 723.424399ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.328817504Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.417195655s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.376108491Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 778.26591ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.387819183Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 650.268667ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.438771686Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.425272524s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.584985127Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 866.898511ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.597386793Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.023885345s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.674413075Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 562.510514ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.708773815Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 558.509363ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.735045408Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:04.802705607Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.498858719s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.877270517Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.147181405s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.890419369Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:04.890537782Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 966.434674ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.909036564Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 979.990849ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.92049805Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 727.422024ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.932289684Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 665.917688ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:04.984652746Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.47266213s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.066524638Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 978.123834ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.10339813Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 33.832428ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.172690752Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 520.157096ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.185519367Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 516.926096ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.216245302Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 146.772834ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.224151965Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.151605925s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.239607435Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.0922969s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.247334275Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 973.953098ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.283618754Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.476898181s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.303623998Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 161.159585ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.379993885Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 553.869703ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.479489461Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 570.840686ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.552860357Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.003858904s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.665388523Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.493876945s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.67688766Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 568.614234ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.712364864Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.486099824s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.725181638Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.019873927s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.744063048Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.104457598s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.802851143Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 580.185961ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.884830777Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 681.238456ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:05.909697921Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 689.995973ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.020681915Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 550.037012ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.043525617Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 690.045345ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.053303959Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 502.770654ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.091495478Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 1.162100344s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.133748191Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.054040247s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.269624119Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 854.314215ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.291223335Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.380279681s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.35093456Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 919.027022ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.385937253Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.023106724s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.399181396Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.398656519s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.474093344Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.055592585s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.548466181Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.033428965s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.600409364Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 898.844861ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.607872298Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 1.350671542s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.616114299Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 507.231941ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.679129731Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.23677331s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.720247231Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 876.076577ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.752438246Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 543.33227ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.773785767Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.478754827s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.789944691Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 972.064075ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:06.916907775Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.27567698s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.156308522Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.262009636s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.181566508Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 614.437379ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.196327878Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.188085393s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.221020939Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 644.456563ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.306652029Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 856.254573ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.315175744Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 967.403573ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.326964848Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.270261978s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.335035815Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.173958439s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.433607471Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.38037533s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.532654807Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 635.019784ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.615909718Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.32863294s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.638594587Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 516.852162ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.648538622Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 925.088144ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.742544095Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.184608945s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.822780652Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.470426539s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.883917746Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.109712849s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.899331714Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 791.321978ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.963245135Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 641.800998ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.971785001Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 584.441056ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:07.982359937Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:08.041487201Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52999"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:08.090385232Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"1744c007-55a4-4d3a-8e3b-8de137856dac","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52999","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":14249,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.395324762s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.176844758Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 570.477935ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.185960777Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 560.419455ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.261750843Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.366052633s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.280466707Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.030911842s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.304553193Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 579.807458ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.420083644Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.085936113s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.441133809Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 994.031238ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.451724957Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 733.743082ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.578040247Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 939.59055ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.61487766Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.319801672s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.623405295Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.4648891s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.632325029Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.15650716s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.677761978Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 905.175641ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.766484401Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 791.415149ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.774837764Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 815.166903ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.832886984Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 542.668772ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.923443665Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.488827424s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:08.978962609Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.271634176s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.01031857Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.127690936s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.019270005Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.240766731s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.02736879Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 1.117624762s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.322576162Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 621.90334ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.346316343Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 787.900547ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.363983228Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.182050853s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.45557265Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 597.040352ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.476038913Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.211297474s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.537920411Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.089107047s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.566378219Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.207113169s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.57708222Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.224330646s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.585631467Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.191183032s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.656114043Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.080198586s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.664025497Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 579.618512ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.680674501Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.11532396s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.846215451Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.482387019s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.95360287Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.078431516s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:09.977598516Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.0190716s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.0852054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.114649459s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.109213006Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.025882134s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.155800225Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 570.206278ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.162898155Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 558.165418ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.27007125Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.005957424s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.278143476Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 701.738731ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.291328219Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 550.424716ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.451925317Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 684.488788ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.47677784Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 714.379088ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.648124901Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.269362621s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.664953919Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.017961649s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.743168555Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.386198105s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.756098162Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 834.681057ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.766366224Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.444688382s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.794818473Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 784.771671ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.818744054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 802.707084ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.836159482Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 889.900036ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.856617489Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.006745645s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:10.972457833Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.357436661s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.001998113Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 591.182068ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.012383668Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.318617186s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.065844552Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 679.779472ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.114883356Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.035323861s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.173962657Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.398213047s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.193740675Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 882.964652ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.233541538Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 638.126064ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.294491137Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.307290387s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.373291375Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.037837941s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.447064449Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.017690727s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.612334927Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.493097819s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.621753559Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 701.773958ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.630081731Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 753.547647ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.648944985Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 753.112354ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.758523635Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.209560226s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.771424481Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.055150759s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.805598477Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.289091644s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.944492558Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.212738682s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.954363682Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.424690021s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:11.990338845Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.293823189s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.129855988Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 706.639754ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.152500786Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.244117242s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.219281296Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.287004943s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.250397259Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 775.750585ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.345154837Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 550.683353ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.371786741Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 998.216792ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.401286806Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 1.344818444s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.419983607Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.078940459s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.498212779Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 543.90865ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.525275468Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 592.742405ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.606326882Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.022538078s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.644007571Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 606.952615ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.694229465Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 713.944788ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.870878699Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 533.938157ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.881628191Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.059993369s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.934545204Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 864.383587ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:12.992085853Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.413095672s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.079736595Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.311495178s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.124641492Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 744.536901ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.133095187Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 872.576498ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.179084077Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.135587574s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.209497186Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.125494959s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.244293569Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 664.618009ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.313374312Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 994.236455ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.411464951Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.36756352s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.427432378Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 678.749922ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.439201681Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 1.403388985s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.449490804Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.437084371s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.47544739Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 755.634719ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.549377408Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 510.349599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.589792492Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 680.767013ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.599046404Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 514.666715ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.676229769Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 878.359893ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.775861308Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 794.519047ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.869475462Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 521.533534ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.88915794Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 530.496323ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:13.991992744Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.143193452s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.005227677Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.440510057s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.064035962Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.380588365s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.113057955Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 661.391349ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.128527435Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.334220125s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.202192497Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.417708603s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.292230948Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 842.538797ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.315474948Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 944.913571ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.358574699Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 546.767871ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.384177829Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 674.455107ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.423387707Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 639.753324ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.44141275Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 996.243778ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.456070995Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 709.591772ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.504905222Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.349769906s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.536354641Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 946.768793ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.665857397Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 903.327018ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.674478335Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 991.928665ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.803777697Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 570.657593ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.815306325Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 620.235556ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.86731418Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.247500404s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.92393741Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 583.209173ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:14.9418354Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 1.096079157s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.093700468Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 1.347174365s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.109158648Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 1.290472393s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.158434716Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 723.146961ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.167941382Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.342439096s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.225469961Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.265339305Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"46.568453541s"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.353738842s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.317044594Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.444129172s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.398310802Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 625.986563ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.465412109Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 519.146221ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.505707242Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.306571227s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.520544918Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.21691444s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.534362754Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.311914586s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.546393162Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 750.665165ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.558790319Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 981.828079ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.594348393Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 1.150287571s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.62468389Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.205584759s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.723952632Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.280615019s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.812851349Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.335043701s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.902706686Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 762.357315ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:15.915417568Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 754.963518ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.041491684Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.145928095s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.112863749Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.07410458s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.188536292Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 735.56977ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.206276049Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 1.032799761s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.32827622Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 550.310364ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.470052009Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 521.211148ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.486171043Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 1.162736416s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.587062517Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 975.775432ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.608630513Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 685.561878ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.690370792Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 1.068634747s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.707896714Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.362052992s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.767945794Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 1.144885015s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.789031241Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 805.569818ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.809816811Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\") failed: Property not found. retry in 1.382445498s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.843786873Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 657.258159ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.860219172Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.188355932s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.8731618Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.060359816s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.944836701Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 751.191903ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:16.960572166Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb\") failed: Property not found. retry in 708.548655ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.024578309Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc\") failed: Property not found. retry in 797.67404ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.035051105Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 798.979539ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.105710685Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.312785084s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.254830707Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.178237464s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.274711268Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.498782369s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.287224129Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b\") failed: Property not found. retry in 908.985604ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.376241695Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.01218689s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.394898631Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 837.96945ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.535754572Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 827.446064ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.603006818Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d\") failed: Property not found. retry in 791.274499ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.631127062Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 829.377024ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.728676078Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 688.016083ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.774681379Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 899.313188ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.791860474Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 960.244287ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.922582965Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 721.19885ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:17.949314298Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.323285668s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.02298193Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 861.947409ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.083249606Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 766.188041ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.151354253Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45\") failed: Property not found. retry in 1.383944494s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.387733317Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3\") failed: Property not found. retry in 1.442082711s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.423527527Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121\") failed: Property not found. retry in 1.199938155s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.459772096Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 1.259021619s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.472676512Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 892.700859ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.485585759Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 1.055294136s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.571348311Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 941.596026ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.584016003Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 661.837187ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.683199323Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 741.932398ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.70474129Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c\") failed: Property not found. retry in 1.074583635s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.799059497Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60\") failed: Property not found. retry in 1.493971776s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.90178376Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d\") failed: Property not found. retry in 1.066204414s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.931194148Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4\") failed: Property not found. retry in 1.030042242s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:18.962660888Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948\") failed: Property not found. retry in 1.364371499s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.359507399Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d\") failed: Property not found. retry in 1.317376705s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.370959855Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf\") failed: Property not found. retry in 830.893659ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.397140185Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0\") failed: Property not found. retry in 900.815208ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.460314931Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48\") failed: Property not found. retry in 1.100810672s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.539560462Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4\") failed: Property not found. retry in 664.22446ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.643960586Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188\") failed: Property not found. retry in 773.951603ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:46:19.748773752Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763387209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_ea2b5fec-1bb6-4488-9cf9-d60d9b99619b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.76345619Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_c3da9028-2f6b-4199-8d06-98c01f75b9bf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.76346719Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_0103f3de-a918-4681-9bb2-af8918749a48"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.76347402Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_598f0700-44ae-48e4-9f3a-a3bd8eeee121"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763480651Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_6c9bae27-e783-49ed-b29e-bd849c13662c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763487071Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_09f2b2d1-ef56-4155-adc5-7f7fd3fc7188"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763493581Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_1ca2a363-9d36-44f3-8de4-33bd220b1a2b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763500121Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_435c2cd2-2bd6-43cc-891d-385af930c2bb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763506421Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_clickhouse_84e31c38-00f6-4075-a1bb-afeeca692948"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763515641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_3a74ab15-bc45-488a-93c4-dcadf3fc9fcc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763521472Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_71af5a4c-a704-4cf4-9d94-94819523084d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763540942Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_ff177c58-abbb-4bc0-bb54-67a185638e45"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763546852Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_c6a4db90-5649-4dbd-b94c-e7b51b493ee0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763557882Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_oximeter_be0f5b06-b4d4-43f1-a81e-d1635992ea60"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763564792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_external_dns_5cab3305-2cf8-4205-93bd-c0a5ec70688d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763570522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_9ff0f763-588f-4b83-b5e3-b5b856e94da4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763579823Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_external_dns_e2d2fc82-684b-4404-ae54-ce48e33d41e3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763635534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_nexus_0339f029-6c8d-4b23-83cc-6885965f777d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.763642694Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3273","zone":"oxz_crucible_pantry_bb25f5f0-a387-4783-a156-b69bcd7dd4e4"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.765245737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.769587907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.771433385Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9bd25fbc-923d-48e2-9efd-8b47c3962c31","remote_addr":"[fd00:1122:3344:101::1]:39995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43236596,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.786581798Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1331"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.786988696Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.787210231Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.80167726Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:708"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.803945567Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.857975473Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:896"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.867339097Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:20.994169538Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:21.159713139Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:21.257300155Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:21.345761244Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:21.471200395Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:21.698667465Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:21.861085242Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:22.079545707Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:22.448699474Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:23.160259479Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:23.791801826Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:24.309249132Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:25.587109348Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:27.157137851Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:28.058915316Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:29.091208218Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/5bbfcf9f-0313-44b5-8699-fd964d8a4a29/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:30.513228512Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"65a5e0ad-7547-4821-90e9-ccb917e4a575\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:33 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"65a5e0ad-7547-4821-90e9-ccb917e4a575\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:34.688297232Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e266971c-f31c-4c3b-bf96-4b1b3ff1b70b\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:37 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"e266971c-f31c-4c3b-bf96-4b1b3ff1b70b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:37.937465169Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:38.128249912Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37920"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:38.128481856Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"2690bbd7-58e4-454d-a73c-6f45ba2a8af5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37920","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4011e72b-1028-4e70-8a4d-ad2eced992c7\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:40 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"4011e72b-1028-4e70-8a4d-ad2eced992c7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:40.182345419Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"097e4a2d-d905-40b3-ad4b-ac36190a6280\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"097e4a2d-d905-40b3-ad4b-ac36190a6280\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:44.628343077Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"835d7d8a-8db5-4080-a018-96b0cfb9c9dc\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:49 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"835d7d8a-8db5-4080-a018-96b0cfb9c9dc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:49.639178668Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0db9a578-4f6b-4ff8-938e-3e92f919550e\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:52 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"0db9a578-4f6b-4ff8-938e-3e92f919550e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:52.717190267Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"892447f7-5ce1-496f-bd96-471a1ef90722\", \"content-length\": \"173\", \"date\": \"Sat, 09 Nov 2024 02:46:57 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"f032ba4a-ac68-41c0-a068-00d3371df648\\\"\", request_id: \"892447f7-5ce1-496f-bd96-471a1ef90722\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:58.229149195Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:958"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.097749745Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.19427089Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63252"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.196018446Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.196249331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.196294692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.198238332Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:46:59.391798712Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"0f455f05-79d3-45f7-b682-7fa74004ef5d","remote_addr":"[fd00:1122:3344:101::b]:63252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":197071,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.108671205Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.109540683Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6b78895a-a3b7-4881-a204-fbace4749131","remote_addr":"[fd00:1122:3344:101::c]:45324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":763,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.148918297Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.148954998Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"35e2d407-e28d-44cd-9958-b44648792459","remote_addr":"[fd00:1122:3344:101::c]:45324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1836,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.201716628Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40671"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.202596766Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ef564785-9b88-484c-b463-edbd328591d5","remote_addr":"[fd00:1122:3344:101::c]:40671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":793,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.326382164Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a42c7fda-8ee0-46b1-b6bd-21d1c130a951","remote_addr":"[fd00:1122:3344:101::c]:45324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.335562474Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36660"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.335664406Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"930f2b2a-2fc0-452b-a520-8236bd4bd29d","remote_addr":"[fd00:1122:3344:101::b]:36660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.371777732Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.371934745Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d26f9f6b-0899-4e63-8542-69199134fab2","remote_addr":"[fd00:1122:3344:101::a]:58661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.372553738Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.37264939Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"07c97fe3-cbe0-45e1-b73f-475c23f713d3","remote_addr":"[fd00:1122:3344:101::a]:34654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":28,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.376245724Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.376260555Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"889d3e8e-8a52-41e4-91e0-76f307c84f31","remote_addr":"[fd00:1122:3344:101::b]:36660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.386822173Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e1cd3f2b-04ce-47ac-80b2-a9497a1df40e","remote_addr":"[fd00:1122:3344:101::b]:36660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.433787804Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.433820054Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"78bf0169-3d5e-430d-a860-ed30d10950ac","remote_addr":"[fd00:1122:3344:101::a]:34654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.447915746Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"f831f0d9-80f8-4cb1-b0de-6401f4a78c81","remote_addr":"[fd00:1122:3344:101::a]:34654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":149,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.485663566Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:03.485850379Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d04a1d92-95d0-4ca9-9ee0-91617990b815","remote_addr":"[fd00:1122:3344:101::b]:34014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:05.266208963Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:41739","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:36579"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:05.326972499Z","hostname":"buskin","pid":1572,"uri":"/f032ba4a-ac68-41c0-a068-00d3371df648","method":"GET","req_id":"933e982f-2aee-4903-9369-30003afae228","remote_addr":"[fd00:1122:3344:101::d]:36579","local_addr":"[fd00:1122:3344:101::1]:41739","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59832,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.16548771Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.165596992Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"96751992-31f2-4a30-9375-25650fa12753","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39266","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.559919691Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64459"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.562169788Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.562229959Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.562248439Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.56225484Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.56226601Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.56227779Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.56228671Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.562312651Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3d7df210-49d9-4eb8-96c6-3128534e2b11","remote_addr":"[fd00:1122:3344:101::c]:64459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2320,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.563079526Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:968"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.753781048Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61987"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.754073444Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.754083584Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.754118115Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.754477002Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:08.957272533Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"1234bf2c-0450-4306-926b-157d369b2962","remote_addr":"[fd00:1122:3344:101::c]:61987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":203365,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.92780722Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36482"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928065596Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928124497Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928158668Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928166428Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928173138Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928188008Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928206299Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.928234559Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d29098ce-0455-45b9-b662-c6498f80884d","remote_addr":"[fd00:1122:3344:101::c]:36482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":331,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947275363Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40972"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947479167Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947521678Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947536678Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947544188Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947560289Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947580289Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.947592629Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:09.94760479Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e275dfd2-6c35-433e-a6a9-2dc1056535a7","remote_addr":"[fd00:1122:3344:101::b]:40972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":229,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.105742788Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42348"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.105958642Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.105995683Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.106038064Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.106061254Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.106069285Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.106074995Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.106080885Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:10.106086585Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"43d632c7-3182-4bbc-9504-64b59b4cdfa3","remote_addr":"[fd00:1122:3344:101::a]:42348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:14.319034441Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:14.319173464Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"cae3dbe7-0243-4f16-b814-ad685e2617b0","remote_addr":"[fd00:1122:3344:101::c]:38789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:14.365379259Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:14.365406019Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c012d380-cce0-4d25-8430-4aee9825e73c","remote_addr":"[fd00:1122:3344:101::c]:38789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:14.373031076Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3efd4866-8ba6-4a2e-ae88-6117df2f6055","remote_addr":"[fd00:1122:3344:101::c]:38789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.266012048Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.266174091Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2fe441e7-2ad5-44aa-a350-924cc6c9b505","remote_addr":"[fd00:1122:3344:101::a]:49320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.28738375Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.28740714Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3bc4a529-aa24-4be4-aa40-8270e85fbd6a","remote_addr":"[fd00:1122:3344:101::a]:49320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.308347713Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38916"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.308510926Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"66a28842-292d-4790-9e42-f1ef5de87a29","remote_addr":"[fd00:1122:3344:101::c]:38916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":54,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.346128234Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.346427928Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"13eaa351-1ff4-4f4a-a245-0be91c850690","remote_addr":"[fd00:1122:3344:101::b]:38382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.356234222Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.356258413Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cc287aff-a4f2-4d6b-8fb6-457b32c5e4c6","remote_addr":"[fd00:1122:3344:101::c]:38916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.373657212Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"bcb361fd-9cf7-4c9e-8ede-b3b153d4bc83","remote_addr":"[fd00:1122:3344:101::c]:38916","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":268,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.41760191Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.417659191Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"bf364d3d-cb23-4b27-892c-9ed2aee1a3d7","remote_addr":"[fd00:1122:3344:101::b]:38382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.429433705Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"25425ec9-d476-4fa3-8192-501d75ff8fac","remote_addr":"[fd00:1122:3344:101::b]:38382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":321,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.604565474Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.604756718Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ca2dd7a9-9afb-4c25-a860-25c211720e77","remote_addr":"[fd00:1122:3344:101::c]:51214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.606556985Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.606658737Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"5878726d-fa9a-4944-9066-907eadf3692d","remote_addr":"[fd00:1122:3344:101::a]:65035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.611400265Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e670f788-a4c4-4bfa-b57b-d0f1e28754a9","remote_addr":"[fd00:1122:3344:101::a]:49320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":746,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.660205344Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:35283"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.660579722Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.750786836Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.817902496Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fa37cd8b-87f7-417a-b8a9-24502094c2eb","remote_addr":"[fd00:1122:3344:101::c]:35283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157670,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.818071061Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:33540"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.818113634Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53209"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.818433044Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.82261036Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c662d850-fbae-40e9-905e-0a9d35b1adab","remote_addr":"[fd00:1122:3344:101::b]:33540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4101,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.823214212Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"971d57bb-d7e2-4525-8123-15d35609cde8","remote_addr":"[fd00:1122:3344:101::c]:35283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4848,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.859919891Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.860869561Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.861411422Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.862617117Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.862674758Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"05d9ad05-d60f-4193-b434-49ba55539dd0","remote_addr":"[fd00:1122:3344:101::c]:35283","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4446,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.870886518Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.927539899Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55334"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.949492882Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.949549383Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2f8c47c9-fd9a-466f-a744-f248551ba58d","remote_addr":"[fd00:1122:3344:101::a]:53209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131218,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.954429944Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5381bd77-f713-49d4-a7b9-b9f8d1444c4a","remote_addr":"[fd00:1122:3344:101::a]:53209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":929,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:33.991074512Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:34.054801918Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1d533ed6-c5f9-496f-a4db-f3607d997321","remote_addr":"[fd00:1122:3344:101::b]:55334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126853,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:34.055236098Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"03edff10-8708-4759-9a13-613e210974c3","remote_addr":"[fd00:1122:3344:101::b]:55334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:38.198579773Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39792"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:47:38.198699346Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"996b21ff-6b37-4ba8-a2a4-2fcd58338d8e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39792","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.239761365Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:34774"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.239917089Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d7a352e8-2a44-427b-80c3-aaa5da91d8bf","remote_addr":"[fd00:1122:3344:101::c]:34774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.258963142Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.258992343Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"72b9905d-76f2-4f6d-bbd9-807d32b687e1","remote_addr":"[fd00:1122:3344:101::c]:34774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.269094902Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a8f2839e-8367-4042-be0e-95b5c3f5eae5","remote_addr":"[fd00:1122:3344:101::c]:34774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.370684641Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38031"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.370761872Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4c12f04a-d751-4d0b-9a41-853ebd33ab74","remote_addr":"[fd00:1122:3344:101::b]:38031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.385497977Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.385686851Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d86d2550-084c-40d6-8a83-ae202dc015e0","remote_addr":"[fd00:1122:3344:101::c]:64138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.401253752Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.401373075Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"eb31e8b3-e292-4b92-9f66-fce989bcd56e","remote_addr":"[fd00:1122:3344:101::a]:54897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":37,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.417215522Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.417243283Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6238e4ef-43f7-47c2-aa0e-444a876fd173","remote_addr":"[fd00:1122:3344:101::b]:38031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":44,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.427228789Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"76087155-12fb-477a-9528-5eba61583451","remote_addr":"[fd00:1122:3344:101::b]:38031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":190,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.43549695Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34674"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.435707574Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"797fa650-4473-4173-bf68-48d8fdbdf7d4","remote_addr":"[fd00:1122:3344:101::b]:34674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.444732751Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.444752071Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b5b85e79-7bd2-4dd6-b8d4-1eadbf42a603","remote_addr":"[fd00:1122:3344:101::a]:54897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.449000839Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42421"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.449088951Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.490291542Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.552044689Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bb6ad18a-e695-4fd8-8b2b-c08211474ffe","remote_addr":"[fd00:1122:3344:101::c]:42421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102986,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.552334405Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41409"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.552415736Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:43771"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.552546629Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"364ca804-a644-4a03-91d1-547c6c9460e3","remote_addr":"[fd00:1122:3344:101::a]:54897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":308,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.55258702Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f6c1b17f-fa71-4116-b358-a10e8bb64f39","remote_addr":"[fd00:1122:3344:101::c]:42421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":261,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.552808754Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.558337919Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"735ef811-71bd-47a1-8ff7-cc212a7ddada","remote_addr":"[fd00:1122:3344:101::a]:41409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5633,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.59469299Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.594778092Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:57233"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.65516072Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.655202701Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2e7be5a4-d896-4b91-8d1a-21fa437e9ff6","remote_addr":"[fd00:1122:3344:101::b]:43771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102502,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.660244955Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"becfba90-3d91-4b26-8b54-72b3da055517","remote_addr":"[fd00:1122:3344:101::b]:43771","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":648,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.691643624Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.74663075Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3510338e-c40b-485c-9d38-bfcaa2aaffa2","remote_addr":"[fd00:1122:3344:101::a]:57233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":147733,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:03.746999138Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c046b618-f12a-4b41-849c-143563dadb07","remote_addr":"[fd00:1122:3344:101::a]:57233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:08.233085467Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57556"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:08.233197689Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"204ea136-fb32-44d6-80d6-402cb6d763a7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57556","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:32.455693677Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.273681912Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.273854585Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"4ab0cdbf-b61b-4fb5-a937-fe432b831f3f","remote_addr":"[fd00:1122:3344:101::c]:64475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.356517253Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.356548104Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7d678fde-d166-4fe3-a5b1-8a94e3f2e6f9","remote_addr":"[fd00:1122:3344:101::c]:64475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.367892599Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ef2e2841-c6d3-49fd-a1a7-514e1116f328","remote_addr":"[fd00:1122:3344:101::c]:64475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":302,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.452307712Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.452392404Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"04abdc39-d095-4158-9b50-6cf76429385c","remote_addr":"[fd00:1122:3344:101::a]:50697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.48171517Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:43453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.481862123Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"79f6b408-3ca9-458d-9ac8-b6077b570b83","remote_addr":"[fd00:1122:3344:101::b]:43453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.50105013Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.50107252Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4f35545-0d09-4123-affd-f58e4998a9d3","remote_addr":"[fd00:1122:3344:101::a]:50697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.505002251Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44519"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.505302398Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"74564a79-4432-4e1f-afb7-dc852898564c","remote_addr":"[fd00:1122:3344:101::c]:44519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":206,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.510878993Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.510899593Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c59e074a-6618-4fcb-8dc6-d4f6fc2841d3","remote_addr":"[fd00:1122:3344:101::b]:43453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.511368003Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"71a9a132-81cd-4c35-8be5-27e08cecae6a","remote_addr":"[fd00:1122:3344:101::a]:50697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":236,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.520177815Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"500aa064-e0c3-443a-82e4-87c3618ac413","remote_addr":"[fd00:1122:3344:101::b]:43453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":284,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.616665719Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38502"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.616793172Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.66028855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.71977517Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9af69a20-57e4-49c0-b572-68407146861d","remote_addr":"[fd00:1122:3344:101::c]:38502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103037,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.719911593Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58202"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.719952734Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49755"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.720002845Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57307"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.720055456Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54057"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.720296971Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.724232022Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"fb40cf03-af46-426a-bf26-208cd70db2d9","remote_addr":"[fd00:1122:3344:101::a]:49755","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3937,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.724366445Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9c05440c-0a67-46c7-af3d-1842f0bcba72","remote_addr":"[fd00:1122:3344:101::c]:38502","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4033,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.724550929Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"62ed55ba-2404-4787-91ce-405f976183e6","remote_addr":"[fd00:1122:3344:101::b]:58202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4173,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.757096401Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.808645636Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.808708948Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b4d45e00-c142-42b2-afcc-a8198f81b926","remote_addr":"[fd00:1122:3344:101::a]:54057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88535,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.812473266Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"565a99d1-0d3b-4e14-b3c4-ea7e0e39b570","remote_addr":"[fd00:1122:3344:101::a]:54057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":619,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.842982356Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.89883812Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a9c6643e-bd51-4b16-8bf7-5dd0630eab41","remote_addr":"[fd00:1122:3344:101::b]:57307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":178417,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:33.899250529Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e7bffdfe-c9ee-423b-8a76-8e865c7244b8","remote_addr":"[fd00:1122:3344:101::b]:57307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:38.273714671Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34125"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:48:38.273855705Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ea8dc252-53fe-4370-a99d-e944e4a296ef","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34125","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.213718828Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57850"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.213936293Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"24890b2d-19de-48d5-bbbb-6ee9c0924fc4","remote_addr":"[fd00:1122:3344:101::c]:57850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.26410871Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54253"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.264233512Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"f378435c-5a6d-4d26-9e0b-d93d1f4fa092","remote_addr":"[fd00:1122:3344:101::a]:54253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.269519271Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.269533982Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a25434b2-5fb1-431f-a709-ee9b874e0392","remote_addr":"[fd00:1122:3344:101::c]:57850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.281625822Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0341c081-78b6-4cb2-ae94-ca855c32a839","remote_addr":"[fd00:1122:3344:101::c]:57850","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":248,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.31443614Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.31445239Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"aab3b806-a81a-4721-8c69-6e9c0b28e272","remote_addr":"[fd00:1122:3344:101::a]:54253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.324552439Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"69040fd2-99f5-415f-bb26-e370631c1f0c","remote_addr":"[fd00:1122:3344:101::a]:54253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":196,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.333258539Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51995"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.333435332Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c13a14d4-563a-48a9-81d5-4abe79c00f4b","remote_addr":"[fd00:1122:3344:101::c]:51995","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.380416313Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42221"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.380526185Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"eb4aa734-3d71-4b7f-96b3-2a71377cad45","remote_addr":"[fd00:1122:3344:101::b]:42221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.391371019Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47416"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.391595624Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"cd788723-ecac-43df-8c71-83c7dade4533","remote_addr":"[fd00:1122:3344:101::a]:47416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":152,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.395029255Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.395044125Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e8716f42-ede7-4c8b-951b-4b5f71b0b5cb","remote_addr":"[fd00:1122:3344:101::b]:42221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.403818546Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0f8ec32c-b381-4bf0-88cb-16aa777a96ec","remote_addr":"[fd00:1122:3344:101::b]:42221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.412830293Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36444"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.412938585Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.454934043Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.511344809Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e0ceacdd-224e-4404-a70a-1af478864435","remote_addr":"[fd00:1122:3344:101::c]:36444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98422,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.511490902Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64125"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.511546533Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.51187651Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c95e2b84-d43c-4570-9041-c19b3ac9d39e","remote_addr":"[fd00:1122:3344:101::b]:61411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":249,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.511978412Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.512016362Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"61117d59-0c3b-49a1-9101-1b7635581a31","remote_addr":"[fd00:1122:3344:101::c]:36444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":290,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.551387026Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.563219501Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50960"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.607802332Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.607865703Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"42d1b3e8-37ce-4bd1-b397-45ec111f7ada","remote_addr":"[fd00:1122:3344:101::a]:64125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96004,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.611201852Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f58d4919-310a-4e45-bfa4-826523505c0f","remote_addr":"[fd00:1122:3344:101::a]:64125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":603,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.641999859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.696336292Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e24b7fef-2ce1-41c9-8bac-3bcbe76f2a28","remote_addr":"[fd00:1122:3344:101::b]:50960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132977,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:03.696697689Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d364e7a2-65f5-4793-9f1f-e6b62960edbe","remote_addr":"[fd00:1122:3344:101::b]:50960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:08.30768152Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:08.307809512Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3cde4160-22f2-46ef-8f36-d637b9780038","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43669","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.272175452Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.272372456Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a317e18f-aa5d-413f-ad44-78de5640856a","remote_addr":"[fd00:1122:3344:101::c]:38624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.336026582Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.336157574Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"324c020e-74f0-4857-ba2d-0d58b55ba96f","remote_addr":"[fd00:1122:3344:101::a]:42172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":64,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.350239025Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.350270026Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0aff7128-9b37-43af-bff1-e26f6b707b7d","remote_addr":"[fd00:1122:3344:101::c]:38624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.364740855Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c86ed302-3ce5-4655-9d20-c8a569e78765","remote_addr":"[fd00:1122:3344:101::c]:38624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":334,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.388987415Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.389007766Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2d1adb82-44df-46ed-ada5-d9f7998b1652","remote_addr":"[fd00:1122:3344:101::a]:42172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.397386689Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e288677f-7fa7-43d5-9882-135b9e7a85ec","remote_addr":"[fd00:1122:3344:101::a]:42172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.40567201Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.405843774Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"371fa588-f399-45d1-a0a1-66f960b5e6fd","remote_addr":"[fd00:1122:3344:101::b]:46481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.445762959Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44061"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.445982723Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"32b31afd-84e0-42a3-a8e1-a4da6da3a8ac","remote_addr":"[fd00:1122:3344:101::c]:44061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.468143491Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45670"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.468251633Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.508609918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.568332972Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"426f9fb1-5cc6-4856-a06b-959c91876497","remote_addr":"[fd00:1122:3344:101::c]:45670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.568464175Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:32861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.568664739Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"cfad6a6c-a8c8-4a8e-b43c-c2b7b25411d2","remote_addr":"[fd00:1122:3344:101::a]:32861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.568804731Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2119f79f-ed4a-4f6f-b01f-a5daa0f2aeca","remote_addr":"[fd00:1122:3344:101::c]:45670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.588913277Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59058"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.58905355Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.623976162Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.679142912Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f70ef17a-5d14-4f23-8d64-4f2ab94d174f","remote_addr":"[fd00:1122:3344:101::a]:59058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90155,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.679383457Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35018"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.679425328Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:33714"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.679439348Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e21a68eb-fcea-4138-9c32-70ce3d7efe52","remote_addr":"[fd00:1122:3344:101::a]:59058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.679664013Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"272a9163-3a84-4cfb-b343-80de2d3e68bb","remote_addr":"[fd00:1122:3344:101::b]:35018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.679675013Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.722040458Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.727568553Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.727610584Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ae703ad8-89ad-42d5-ad52-d5b795616fee","remote_addr":"[fd00:1122:3344:101::b]:35018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":133,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.738020599Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b0934d46-d095-4a2e-8471-25d0aa5f565f","remote_addr":"[fd00:1122:3344:101::b]:35018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":404,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.778043576Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"14abfc8f-4310-4d38-9943-4a4b615162be","remote_addr":"[fd00:1122:3344:101::b]:33714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98475,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:33.778349952Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"862f9d7e-1f9f-4c65-96e7-92203186471d","remote_addr":"[fd00:1122:3344:101::b]:33714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:38.344265101Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40845"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:49:38.344952015Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"8109b5b0-8c7e-48c2-bf1c-5abbc5b1ba76","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40845","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.212609207Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42932"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.212779161Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"17c92910-72e1-4cf7-84e6-f89fbc3b7c31","remote_addr":"[fd00:1122:3344:101::c]:42932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.241261369Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.24129691Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"10b24079-1015-4e75-a575-8db38ee70f50","remote_addr":"[fd00:1122:3344:101::c]:42932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.251199395Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ba02440c-c355-4b7b-9569-98494fea1ee5","remote_addr":"[fd00:1122:3344:101::c]:42932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":234,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.277319734Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53080"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.277506148Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c4f1989f-6f45-4834-a10c-9a901262b1b9","remote_addr":"[fd00:1122:3344:101::c]:53080","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.313011292Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54769"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.313157355Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.356186594Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.424622767Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"982a2f82-292f-4c8d-8048-ac47ab8303e2","remote_addr":"[fd00:1122:3344:101::c]:54769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111529,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.42474007Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61280"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.424796791Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.424992095Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"11741af5-6039-4ea0-b4d7-ef13609e4f7b","remote_addr":"[fd00:1122:3344:101::b]:63098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.4252195Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"06848639-b703-4d2e-82e5-fe11700a3363","remote_addr":"[fd00:1122:3344:101::a]:61280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.42524344Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"668d56ea-aca5-4613-9854-b13bac488e18","remote_addr":"[fd00:1122:3344:101::c]:54769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":275,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.437055375Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61089"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.437297409Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"da265861-ee87-4117-9ed7-0462db87f342","remote_addr":"[fd00:1122:3344:101::b]:61089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.440598118Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38049"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.440728151Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.487976017Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.548865365Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9b0b6ba3-ec8a-42ef-9a86-187df2a749af","remote_addr":"[fd00:1122:3344:101::a]:38049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108187,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549147531Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549164552Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549203482Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d1b6ef18-8182-4f2d-a64b-dc721816d835","remote_addr":"[fd00:1122:3344:101::b]:63098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549262774Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47198"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549406486Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549435497Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"41ba9441-b41b-4358-b21e-d7e0de2ccb15","remote_addr":"[fd00:1122:3344:101::a]:61280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":175,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549549369Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0b51e424-8c3d-4d40-8fe0-adb792ddaf80","remote_addr":"[fd00:1122:3344:101::a]:38049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":303,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.54956058Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.549647902Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7b079164-ff04-469b-bdcb-eab23d6f3adf","remote_addr":"[fd00:1122:3344:101::a]:47198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":251,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.559229089Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5c3e9524-e60f-4a75-966e-69d8e4538150","remote_addr":"[fd00:1122:3344:101::a]:61280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":291,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.560345683Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ee733ef5-5fd4-4c28-ac35-ea6e23eb6112","remote_addr":"[fd00:1122:3344:101::b]:63098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.590400554Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.648204438Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2051c39a-65b5-4eec-a10d-7d7d2bb4d2a0","remote_addr":"[fd00:1122:3344:101::b]:36130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98787,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:03.648542975Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"79ee6dd7-9439-4312-92da-f3c1ba6bab45","remote_addr":"[fd00:1122:3344:101::b]:36130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:08.379954264Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49495"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:08.380095317Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"1758c0c4-5f2c-464c-9568-4c1c702a8533","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49495","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.39158416Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61971"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.391733144Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"451ff417-f4cd-4bf1-9d7a-1e4a5645cdbc","remote_addr":"[fd00:1122:3344:101::a]:61971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.450389246Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.450431797Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"49f8bf54-8bb4-46aa-aa1e-ec4662ddb55d","remote_addr":"[fd00:1122:3344:101::a]:61971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.479853305Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39050"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.480031348Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"72ea5931-c482-4715-9f16-86d687327fda","remote_addr":"[fd00:1122:3344:101::c]:39050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.499079722Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.499098713Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e37def96-42e8-49e7-96dd-7b94720f4d47","remote_addr":"[fd00:1122:3344:101::c]:39050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.508474406Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"618b53af-310c-4c66-bba5-94385d27bbda","remote_addr":"[fd00:1122:3344:101::c]:39050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.569959547Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65090"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.570139561Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a190b7b3-1386-437d-b74d-2be5037f39dd","remote_addr":"[fd00:1122:3344:101::a]:65090","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.580546086Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44592"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.580632578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.625100326Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.679373258Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"23853152-79b3-4b8e-87ff-fe01ac174492","remote_addr":"[fd00:1122:3344:101::c]:44592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98782,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.679518301Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63078"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.679553712Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.679902489Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b1ace4b0-51f7-4751-b235-f80c3182d84f","remote_addr":"[fd00:1122:3344:101::b]:41496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":217,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.67993316Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.680112603Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f51d0ce7-b886-4d0f-acf5-516e86e377e7","remote_addr":"[fd00:1122:3344:101::c]:44592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":365,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.680139584Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.680304498Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9bb5b8b4-4a26-40a3-a055-3bc656a78669","remote_addr":"[fd00:1122:3344:101::a]:61971","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":591,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.685951444Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1817a66d-4e71-47d9-a2f8-39962a47f3a4","remote_addr":"[fd00:1122:3344:101::c]:33841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5754,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.715173708Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.715414983Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"530cd0ef-7d4a-40a9-b10b-bf27cf48bb82","remote_addr":"[fd00:1122:3344:101::b]:46172","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":146,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.721430627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.758369241Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.758420802Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7d09cc48-105f-48b6-839e-87d04540e8e1","remote_addr":"[fd00:1122:3344:101::b]:41496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.767956689Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"4d245ab2-981e-42c2-a194-3530b1b37fd7","remote_addr":"[fd00:1122:3344:101::b]:41496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":292,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.781741624Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2cc8a90a-0286-457a-989f-9c2f0bb84391","remote_addr":"[fd00:1122:3344:101::a]:63078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101864,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.782102341Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1bb9d11b-4c0e-4212-8d4c-824b0b6f5aaf","remote_addr":"[fd00:1122:3344:101::a]:63078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.787474072Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54414"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.787641506Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.824579939Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.878632226Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"88657247-dbfd-450c-9496-e2938589e331","remote_addr":"[fd00:1122:3344:101::b]:54414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91059,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:33.878886991Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0c86beb5-f045-40cf-9027-e11177df3128","remote_addr":"[fd00:1122:3344:101::b]:54414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:38.414004553Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:50:38.414092705Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"2302402a-1c6d-4d5e-96bd-e7262c135a35","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34704","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.268078655Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59428"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.268204427Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2b2039de-389c-4bba-965d-61ac7c2aaa0b","remote_addr":"[fd00:1122:3344:101::c]:59428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.288258902Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.288277742Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c9887714-946c-4410-9b43-aa226a2d1cae","remote_addr":"[fd00:1122:3344:101::c]:59428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.298656267Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"dfd42964-ec2b-49ba-ac67-67f14b8a919e","remote_addr":"[fd00:1122:3344:101::c]:59428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":230,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.354262616Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40297"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.35444817Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"df6d4779-03cc-4665-87b5-0483df183abb","remote_addr":"[fd00:1122:3344:101::c]:40297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.3588199Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.358954593Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7f98f4be-131c-4d63-94a4-5b141d8bbdcf","remote_addr":"[fd00:1122:3344:101::b]:39435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.377804262Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54763"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.377984085Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.418416711Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.482351292Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"874303a8-8dbd-48a4-b244-8a8320b0f0c4","remote_addr":"[fd00:1122:3344:101::c]:54763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104431,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.482618228Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39378"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.482917864Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.482941844Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"15068536-582a-4b0e-ab91-c9c4f4820b17","remote_addr":"[fd00:1122:3344:101::b]:39435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":208,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.482994296Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9fcab11b-a98a-44f8-a113-1fe5988f116e","remote_addr":"[fd00:1122:3344:101::a]:39378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.483040706Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8890d9e4-f383-45db-a7d5-2ea9b8fecdc3","remote_addr":"[fd00:1122:3344:101::c]:54763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":301,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.485197441Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63282"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.485366005Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.525481074Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.585384361Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"64821d2b-2b19-464e-8aa0-9310c292a2ce","remote_addr":"[fd00:1122:3344:101::a]:63282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100070,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.585640447Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50510"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.585656097Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.585711808Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7aa2fc2a-ea54-40a2-9128-554852ae619c","remote_addr":"[fd00:1122:3344:101::a]:39378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.585724819Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37726"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.585848971Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45279"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.586055155Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"fbb2a581-08f4-4384-9b3d-59119babbd6e","remote_addr":"[fd00:1122:3344:101::b]:50510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":177,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.586087166Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b43d8a6d-a926-415c-b52a-7abf0512ecfd","remote_addr":"[fd00:1122:3344:101::a]:63282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":301,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.586212029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.586219039Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"18dcb53e-e3e3-42b5-a01c-b587183638b1","remote_addr":"[fd00:1122:3344:101::b]:39435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":439,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.586325061Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ddb66ff0-1747-4927-beaa-358439ba06e8","remote_addr":"[fd00:1122:3344:101::a]:37726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":308,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.596721846Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d0dc4885-98a2-4089-809f-efee0ebf6a65","remote_addr":"[fd00:1122:3344:101::a]:39378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":316,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.626730116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.681728613Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c6ded998-3568-490e-8761-944fd1d244ee","remote_addr":"[fd00:1122:3344:101::b]:45279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95622,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:03.68208556Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"087292ad-1aac-478c-88cb-383d7242854d","remote_addr":"[fd00:1122:3344:101::b]:45279","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:08.448012522Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:08.448117984Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3e05bc5d-88a6-4599-8663-9a3c11d7915b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38299","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.225446559Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44082"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.225626623Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ec31b32a-819a-4c97-8f3d-7dc03b793608","remote_addr":"[fd00:1122:3344:101::c]:44082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":87,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.34501335Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.345044591Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"887aa27b-eb0e-414e-84d6-c6080947a789","remote_addr":"[fd00:1122:3344:101::c]:44082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.366334401Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3bb587ee-f75e-4f3d-a990-c8d2a151f48a","remote_addr":"[fd00:1122:3344:101::c]:44082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.471679698Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.47178095Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ec165461-4598-415d-9ef3-e79dbc5bb925","remote_addr":"[fd00:1122:3344:101::a]:48194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.528290797Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.528320208Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"69558bd9-af9e-43f1-bd30-1266e2120526","remote_addr":"[fd00:1122:3344:101::a]:48194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.564187459Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.564410034Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ad8cb434-d608-400c-b828-69e78def73ea","remote_addr":"[fd00:1122:3344:101::a]:47107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.566675961Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.566847384Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"dd6ce594-413c-4693-8546-72411f92c1e9","remote_addr":"[fd00:1122:3344:101::c]:42191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.576544535Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1b3e3df6-2e60-46ac-9a69-f6d443617ab5","remote_addr":"[fd00:1122:3344:101::a]:48194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.578320571Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.578434044Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5169aa89-dfc1-44a4-802d-f6bc7f7fc284","remote_addr":"[fd00:1122:3344:101::b]:51960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.606769909Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43423"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.606909732Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.653392863Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.721922239Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cc6c57c1-5436-4aa0-a82e-debd334a69a2","remote_addr":"[fd00:1122:3344:101::a]:43423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115061,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722098223Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47866"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722181755Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36084"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722486721Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722538022Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ff7c425d-b7e1-4d9f-b836-45bc39d08478","remote_addr":"[fd00:1122:3344:101::b]:51960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":263,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722548262Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722567172Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"75e9bde7-959d-467a-958e-d12f05ca9826","remote_addr":"[fd00:1122:3344:101::a]:43423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":308,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.722630874Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c47958b1-0eb3-4dbe-9ed2-48ecf9312faa","remote_addr":"[fd00:1122:3344:101::b]:36084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":259,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.733787635Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7f7c7fdb-b7a3-45da-b9ad-78ea3382fb1c","remote_addr":"[fd00:1122:3344:101::b]:51960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":264,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.744989066Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36595"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.772204658Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.790994157Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51259"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.791496597Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.791581989Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79161932Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79163641Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.791700011Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.791711801Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.791723672Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.791755653Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"709e92fa-a301-4068-9953-8e498281c1db","remote_addr":"[fd00:1122:3344:101::a]:51259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":572,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.796264976Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50645"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.796404198Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.796428749Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.796448899Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79645425Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79647636Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79649226Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79649702Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.79650318Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a4480aa3-ef14-43d6-a476-02516f0157f0","remote_addr":"[fd00:1122:3344:101::c]:50645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.806824404Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59075"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.807041758Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.807069369Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.80712182Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.80713979Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.8071469Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.807153371Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.807171841Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.807176921Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2e6f0934-9f69-43d7-8541-bfc80d51e6d7","remote_addr":"[fd00:1122:3344:101::b]:59075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":249,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.838524739Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"da0b1a34-d133-43a3-b47a-9ce763ff26b9","remote_addr":"[fd00:1122:3344:101::c]:47866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116089,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.83858722Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.843639465Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bb07f68d-f711-4bdf-937d-45cf217f597a","remote_addr":"[fd00:1122:3344:101::c]:47866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":837,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.87825266Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.937679158Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"6cd46efc-2c6b-47ac-9eb8-ed6fb0f5ae65","remote_addr":"[fd00:1122:3344:101::b]:36595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192542,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:33.938077826Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0b090ec3-6f23-4e3a-ae0d-bad5790cd18f","remote_addr":"[fd00:1122:3344:101::b]:36595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:38.48008439Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:51:38.480217913Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3a126fc5-5728-4147-bd39-828ce74f0ef7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65457","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.121523477Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.12165366Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"114c332e-6063-40bf-926e-c0b5f03ad876","remote_addr":"[fd00:1122:3344:101::c]:37073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.175265678Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.175295938Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b28b3567-b9f3-45e8-bc45-e2c7c588bfdc","remote_addr":"[fd00:1122:3344:101::c]:37073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.198485978Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6979582c-d766-4ab6-8bdc-e5486542f7a9","remote_addr":"[fd00:1122:3344:101::c]:37073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":257,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.299452804Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42944"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.299564827Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.345732461Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.408450156Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"6fd715c7-139f-499b-a9a1-b80a46ace8f6","remote_addr":"[fd00:1122:3344:101::c]:42944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108929,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.408594179Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50615"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.408656501Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.408992158Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ae4da60d-86ed-4b01-bb7c-15c9a9bd513f","remote_addr":"[fd00:1122:3344:101::a]:41927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.409139841Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7bb882c6-7ca5-496e-9ce6-dc65ef87578f","remote_addr":"[fd00:1122:3344:101::c]:42944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":305,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.409203382Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"852a41a4-8ac1-4787-8abe-98a84b479f28","remote_addr":"[fd00:1122:3344:101::c]:50615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":258,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.425761394Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.425802165Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f9cde6bb-4762-4a9d-9d84-bc24bab6011d","remote_addr":"[fd00:1122:3344:101::a]:41927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.435314162Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"68799608-2b92-485b-bf5a-f6087b606fb0","remote_addr":"[fd00:1122:3344:101::a]:41927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.449614357Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48196"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.449694869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.489223996Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.54461447Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cba43f45-41aa-4188-9d8a-cf170afe9b2b","remote_addr":"[fd00:1122:3344:101::a]:48196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94940,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.544789144Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62639"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.544812635Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.545133911Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c272ed91-c5ed-44c6-9e4c-14473492adc8","remote_addr":"[fd00:1122:3344:101::a]:53836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":223,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.545143892Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"150b2846-f91d-4e47-9f78-ab581be7f621","remote_addr":"[fd00:1122:3344:101::a]:48196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":293,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.545208873Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"67fdaf9f-b35e-48bd-be39-bea00ea74f2a","remote_addr":"[fd00:1122:3344:101::b]:62639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.580103804Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61880"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.580258707Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.621979149Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.683216185Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7680075d-86cc-430c-b466-118f96f73bb9","remote_addr":"[fd00:1122:3344:101::b]:61880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103024,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.68344911Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58401"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.683481111Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.683490401Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"866ad1bf-4612-458c-918f-cf3699cf3841","remote_addr":"[fd00:1122:3344:101::b]:62639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.683741486Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"6cd1bc71-bc1b-4f7f-91e9-a3b320c51da0","remote_addr":"[fd00:1122:3344:101::b]:58401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.683764066Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"1b7dcdf9-344c-487e-ad92-6e9911bdad16","remote_addr":"[fd00:1122:3344:101::b]:61880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":243,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:03.694290694Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c3c59d0a-85d4-4f06-ba25-a394b1d5287c","remote_addr":"[fd00:1122:3344:101::b]:62639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:08.512885414Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:08.513028127Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"bdfcfb4f-9ef4-44b4-824f-1a7aa10d272f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33864","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.329758156Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.32993609Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"42422950-e22f-4c58-8078-c0dc180d31bb","remote_addr":"[fd00:1122:3344:101::c]:58421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.39914209Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.39917964Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"273bf59f-acf0-4a43-bd92-450dc7fe30b3","remote_addr":"[fd00:1122:3344:101::c]:58421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.409195747Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2d495a51-495b-4f16-b27e-ec978915b1f3","remote_addr":"[fd00:1122:3344:101::c]:58421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":235,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.510204385Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.510293107Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e351283f-2bdd-4254-9d27-69e7c2e4ba6a","remote_addr":"[fd00:1122:3344:101::b]:51408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.527319299Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.527607605Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d735cc8b-96a9-4bfe-b47f-590d1962a5a3","remote_addr":"[fd00:1122:3344:101::c]:60597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.533613338Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.533628829Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0d9506c8-13e7-4151-ad6a-b3d395708116","remote_addr":"[fd00:1122:3344:101::b]:51408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.542244697Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a3dd8e27-6d83-4f39-8348-d0f675f15820","remote_addr":"[fd00:1122:3344:101::b]:51408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":149,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.617951921Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61257"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.618034393Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.659068531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.724774349Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5fc61e0f-0e9d-4761-984a-a8c1211d19fa","remote_addr":"[fd00:1122:3344:101::c]:61257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106760,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.724908222Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52286"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.724931312Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.725233659Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"dba7c0f2-e9f3-442f-b745-156331a6390d","remote_addr":"[fd00:1122:3344:101::b]:49068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":211,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.725266899Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"70199779-0ab9-4a6a-a935-192f242df206","remote_addr":"[fd00:1122:3344:101::c]:61257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":226,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.725344891Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c00775ce-3ff4-4fc2-a0be-08a8be6c38a5","remote_addr":"[fd00:1122:3344:101::a]:52286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.768260328Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.768283139Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2d984780-9081-4707-99f3-2645bfe2d401","remote_addr":"[fd00:1122:3344:101::a]:52286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.776802004Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"762d35ec-b7de-4797-967d-f04850a5a8bd","remote_addr":"[fd00:1122:3344:101::a]:52286","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.780389989Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52609"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.780496771Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.827776808Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.889619306Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"84df42ed-f869-4585-ae30-97d990d79799","remote_addr":"[fd00:1122:3344:101::b]:52609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109155,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.889778079Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65125"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.88981281Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49127"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.890092046Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.890130506Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8b01aa32-c0cf-4590-bef3-0ecc8760d860","remote_addr":"[fd00:1122:3344:101::b]:52609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":278,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.894008207Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"23ddc654-1eea-41dd-b09f-02ddb0e77428","remote_addr":"[fd00:1122:3344:101::a]:49127","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3909,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.926423907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.981651688Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"dd5074cb-5664-41d9-be07-23cde032dd11","remote_addr":"[fd00:1122:3344:101::a]:65125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91696,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:33.982026776Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7bd5eeca-d64e-4a47-bf93-d32b2355e549","remote_addr":"[fd00:1122:3344:101::a]:65125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:38.547756099Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49080"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:52:38.547846751Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"393175aa-229d-4210-ae33-a5135b66eefe","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49080","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.199270802Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62913"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.199471836Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d2204427-f88d-4dcc-9efd-396c918ef63f","remote_addr":"[fd00:1122:3344:101::c]:62913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.270881752Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.270913083Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d89a92ca-cae6-4db2-b3d7-e7efe55c895e","remote_addr":"[fd00:1122:3344:101::c]:62913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.288420275Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"8d0a5d45-18f9-4992-9a5b-38d1c71ad539","remote_addr":"[fd00:1122:3344:101::c]:62913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":226,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.292662712Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46397"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.292766474Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.342719257Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.4028934Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7c7be449-5ce6-4e88-9860-a96da99a942a","remote_addr":"[fd00:1122:3344:101::c]:46397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110181,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403113764Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59242"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403166895Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403225256Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a2613e00-9c0c-4103-85f0-866fa796b97c","remote_addr":"[fd00:1122:3344:101::c]:46397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403238287Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42837"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403294028Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55113"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403616665Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c5c41495-39f7-4966-9bb7-d48b9f2bd7fe","remote_addr":"[fd00:1122:3344:101::b]:55014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":163,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403755027Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7ab77dee-6744-4916-9948-0e4d9bea0d8c","remote_addr":"[fd00:1122:3344:101::c]:59242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":373,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403777518Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a9cdf141-a222-4245-a024-798ab7d7230a","remote_addr":"[fd00:1122:3344:101::a]:55113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":301,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.403844899Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b9d28f66-ceff-4ac7-b8d4-32f7208f59b3","remote_addr":"[fd00:1122:3344:101::a]:42837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.419559364Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.419582134Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"dbdd05af-c798-4a31-8941-9a81a73fee95","remote_addr":"[fd00:1122:3344:101::a]:42837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.419727988Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.419735768Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"bb54c226-2c55-4f63-b95b-5a807b6ef3b4","remote_addr":"[fd00:1122:3344:101::b]:55014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":22,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.431748096Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"9a4d71d5-f683-446a-8ece-207a701bc94e","remote_addr":"[fd00:1122:3344:101::b]:55014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":206,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.432535802Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e772fd50-7f63-4701-b4d6-0a61e2b53121","remote_addr":"[fd00:1122:3344:101::a]:42837","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.46002364Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57058"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.460174904Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"85a09a2a-c51d-4a14-975d-9c37d6cd6b30","remote_addr":"[fd00:1122:3344:101::b]:57058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.463823269Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63376"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.463909661Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.502421926Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.557298601Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e16f5ab1-c362-4ea0-8972-fcbcef6484a4","remote_addr":"[fd00:1122:3344:101::a]:63376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93408,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.557436783Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60154"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.557625287Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.561733782Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bbe33217-25e8-4311-a297-638215dbfc58","remote_addr":"[fd00:1122:3344:101::a]:63376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4091,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.591101809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.644062844Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e6aa9232-d101-4d61-9467-60b48ebc51a0","remote_addr":"[fd00:1122:3344:101::b]:60154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86510,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:03.644394361Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a9aaa917-3b5d-419f-a35b-070100028db9","remote_addr":"[fd00:1122:3344:101::b]:60154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:08.580173962Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:08.580299285Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"6c82867b-5eda-49f3-8348-cf80d2293462","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:38522","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:17.593555932Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:17.593623213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:17.62541125Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999584s","next_cleanup":"Instant { tv_sec: 1900, tv_nsec: 963331011 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:32.459008889Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.284353316Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42352"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.28455168Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6aa41539-0b80-4e75-9077-aa5c88f134a1","remote_addr":"[fd00:1122:3344:101::c]:42352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.358002248Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.358038329Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6c762d7b-a087-4d55-bc0e-6bf7ec49b551","remote_addr":"[fd00:1122:3344:101::c]:42352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.369471025Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ae38288e-63d0-4634-897e-5e255aa96c88","remote_addr":"[fd00:1122:3344:101::c]:42352","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":231,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.393690275Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34353"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.393799347Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3fc2d4a0-0965-45ea-888e-2ddecb7cff58","remote_addr":"[fd00:1122:3344:101::b]:34353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.412752559Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.412786509Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"196d523b-afe0-466c-a99b-616dd49d29e0","remote_addr":"[fd00:1122:3344:101::b]:34353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.423001801Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1bf36499-e251-4d0a-acef-8ec336b064f8","remote_addr":"[fd00:1122:3344:101::b]:34353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":194,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.55603063Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45378"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.556195793Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"603c71d8-a386-4886-88c2-89f6aba214d9","remote_addr":"[fd00:1122:3344:101::b]:45378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.646781636Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62871"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.646942649Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"65d8d745-88f3-4ead-8aab-ff4212cc2b74","remote_addr":"[fd00:1122:3344:101::c]:62871","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.6528228Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44330"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.652872861Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d9815325-4cd5-47ad-b63e-7327e552fc9e","remote_addr":"[fd00:1122:3344:101::a]:44330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.668328161Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.668348921Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"f6d697aa-82f2-4023-963f-8c1414b9d7af","remote_addr":"[fd00:1122:3344:101::a]:44330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.669805801Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38740"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.669920684Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.712755479Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.775944835Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"815dea9c-6671-443c-9399-fed7a9ac4b61","remote_addr":"[fd00:1122:3344:101::c]:38740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106066,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.776100948Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55973"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.776374414Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2d3f39bd-25ef-4431-b115-13cefe109ea6","remote_addr":"[fd00:1122:3344:101::c]:38740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":161,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.776488846Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.776510786Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d10fbb51-4c01-4997-966c-0f16ba2b1929","remote_addr":"[fd00:1122:3344:101::a]:44330","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":321,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.798351998Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34165"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.798561892Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"2dcffc35-ff23-49c1-9583-32900e93f4ec","remote_addr":"[fd00:1122:3344:101::a]:34165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":154,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.815750797Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.840462288Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43802"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.878054305Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.878113076Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"e518a767-c158-4f44-8bf7-8c3dd7fe55c3","remote_addr":"[fd00:1122:3344:101::b]:55973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101705,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.884452427Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bf4b1b21-59e0-4f3d-b686-56b3baeb76e4","remote_addr":"[fd00:1122:3344:101::b]:55973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":355,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.919752007Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.977889688Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c809956a-62f7-469b-a7f5-4d8ead33300c","remote_addr":"[fd00:1122:3344:101::a]:43802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":137301,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:33.978238835Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"2ea7d585-5307-4bdc-9768-21ad2cff7998","remote_addr":"[fd00:1122:3344:101::a]:43802","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:38.611666109Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:53:38.611769131Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f5111cec-7fc8-4d5c-ac7d-dc5d6c9d3d5a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47593","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:57.666772285Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:57.667572716Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:57.669516036Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:59.494969891Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:59.495785337Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:59.496550463Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:59.497228237Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:59.497919772Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:53:59.498650527Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.238197398Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45359"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.238348491Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2f55fd6b-f721-46fe-b170-b67746872661","remote_addr":"[fd00:1122:3344:101::c]:45359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.297245718Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.297277939Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5bee3d59-516b-49ee-a92a-1da53a693550","remote_addr":"[fd00:1122:3344:101::c]:45359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.308673394Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"82eb0dc7-7284-43f8-a9ad-0a05a045499a","remote_addr":"[fd00:1122:3344:101::c]:45359","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.316772652Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.316917365Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e443242d-f26b-4b0f-a046-9ecf90d73ddd","remote_addr":"[fd00:1122:3344:101::a]:33375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.348848825Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.349072679Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"2a3330c7-2693-43a6-afb5-2db9fe678dfe","remote_addr":"[fd00:1122:3344:101::c]:58650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.363842945Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.363863555Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0c34c01c-74ba-457d-af93-87b370f7c1c4","remote_addr":"[fd00:1122:3344:101::a]:33375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":53,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.366484619Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41399"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.366568311Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.411507309Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478110655Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f03fb860-0a55-48f6-b855-a0a593ab615f","remote_addr":"[fd00:1122:3344:101::c]:41399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111571,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478408072Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63923"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478490254Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63892"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478552385Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56469"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478634016Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2f500c02-085a-499d-8966-c704b520d3d6","remote_addr":"[fd00:1122:3344:101::a]:33375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":339,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478744469Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"97081ddb-c1a5-47ee-bdd0-96d975b7fd6e","remote_addr":"[fd00:1122:3344:101::c]:41399","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":312,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478863111Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ccd8d7a3-80f9-4d88-a17d-d72cca7f0d33","remote_addr":"[fd00:1122:3344:101::b]:56469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478936733Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"85f724e9-4efa-442f-9f44-aa8ddd328547","remote_addr":"[fd00:1122:3344:101::a]:63923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":222,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.478961653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.496535516Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.496615728Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7bfe1990-484b-41a5-87b6-3ed8c5bef34d","remote_addr":"[fd00:1122:3344:101::b]:56469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.506256707Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d67c32d1-6922-4016-9c4b-989d8e36d268","remote_addr":"[fd00:1122:3344:101::b]:56469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":264,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.52721372Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.538984814Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53450"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.539246839Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"79b41538-1050-4250-8dbf-5f04597e09bb","remote_addr":"[fd00:1122:3344:101::b]:53450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.553243318Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60163"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.595938361Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.595982202Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7e5abb42-1794-4d40-a817-9017b51078ee","remote_addr":"[fd00:1122:3344:101::a]:63892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117081,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.602059367Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ff52f810-c46c-4467-ad73-48cd9e8fab58","remote_addr":"[fd00:1122:3344:101::a]:63892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":816,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.639300507Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.704106766Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"02eceaeb-739e-4ea0-8a43-61f3ed32df82","remote_addr":"[fd00:1122:3344:101::b]:60163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150754,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:03.704519795Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0de754b4-7685-4ced-b45b-2d7145be8a58","remote_addr":"[fd00:1122:3344:101::b]:60163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.666358749Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.668442142Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.669157997Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.669886212Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.670576536Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.671251641Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.671935664Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.672444895Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.672961626Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.673824474Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.674514158Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:07.675169061Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:08.64333962Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:08.643439442Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"668d0d18-c8f9-4b7b-9241-3244cf4f8b8a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55194","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:09.499317149Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:09.500217948Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:09.501056895Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:09.50177939Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:09.502498985Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:09.50322806Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.670503644Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.672588897Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.673374073Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.674131169Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.674865074Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.675617559Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.676346785Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.676863885Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.677397806Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.678092151Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.678816605Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:17.679475119Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:19.504446564Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:19.505304081Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:19.506099658Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:19.506832103Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:19.507560888Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:19.508262832Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.673738929Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.675814612Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.676521456Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.677197761Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.677932296Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.678668591Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.679328334Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.679877936Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.680422227Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.681115111Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.681812436Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:27.682458489Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:29.509392204Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:29.510246472Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:29.511035308Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:29.511754463Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:29.512470428Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:29.513156562Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.40643129Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52052"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.406626434Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1f6bc193-3353-4f52-b0c5-c9414a9f3720","remote_addr":"[fd00:1122:3344:101::c]:52052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.423995963Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.424038634Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"3c7be553-25a1-48d7-a59b-184cf7646ea0","remote_addr":"[fd00:1122:3344:101::c]:52052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.435505401Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1fe08157-81a7-4a53-aba9-15ba797d8f1f","remote_addr":"[fd00:1122:3344:101::c]:52052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":233,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.456112447Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.456227609Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"178cf8a1-4795-4299-9307-a29e98ce93ee","remote_addr":"[fd00:1122:3344:101::a]:53404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":45,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.506851215Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.506880806Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"08baa1d6-9e22-48fe-914f-e678014ecf80","remote_addr":"[fd00:1122:3344:101::a]:53404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.556547022Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47652"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.556786407Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"fddfe182-2f30-45c5-a3ba-edd4c795acb1","remote_addr":"[fd00:1122:3344:101::a]:47652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.569832697Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"321e4c66-7c7d-4361-b19c-67f7e91cb149","remote_addr":"[fd00:1122:3344:101::a]:53404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":215,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.596648571Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60921"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.596816044Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"f2f07366-4a71-4e48-b76b-c1fbd0de0ada","remote_addr":"[fd00:1122:3344:101::c]:60921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.608496326Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59661"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.608600168Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.656522678Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.726948434Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"2452914c-a332-46c8-8606-92e2a4c8ed0f","remote_addr":"[fd00:1122:3344:101::a]:59661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118387,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727094007Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:60955"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727126558Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61889"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727171258Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:40275"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727220739Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37005"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727590197Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727680289Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3975690d-f3d6-4a64-84b6-142ea168c937","remote_addr":"[fd00:1122:3344:101::a]:59661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":341,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.727695149Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"658ae5ed-c129-438b-b292-0f0cf224ffbe","remote_addr":"[fd00:1122:3344:101::b]:60955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.733233804Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d8d0a4a5-710e-4e4a-9c78-c0e9909598f9","remote_addr":"[fd00:1122:3344:101::b]:40275","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5635,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.741426903Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.741446933Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"fe9594be-a0eb-4eaa-8383-6f1544e05b8f","remote_addr":"[fd00:1122:3344:101::b]:60955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.750057071Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b0102c6c-3a09-43eb-bb83-743baf980daa","remote_addr":"[fd00:1122:3344:101::b]:60955","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":338,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.773114718Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.837150851Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.837202583Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"62a1ffd5-55c7-47f8-923b-d1fbc8fe1f53","remote_addr":"[fd00:1122:3344:101::b]:37005","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109823,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.842655335Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"059917e1-4415-4013-b014-32a0f9400edb","remote_addr":"[fd00:1122:3344:101::b]:37005","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":852,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.879027727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.940591389Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"16076451-92e8-4df2-a772-71a837cb176d","remote_addr":"[fd00:1122:3344:101::c]:61889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":213175,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:33.940984897Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a5c5f4ac-d336-4a4a-a12e-c7165e5a007a","remote_addr":"[fd00:1122:3344:101::c]:61889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":140,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.67822271Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.680436666Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.681161845Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.681899826Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.682610911Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.683304075Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.683984499Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.6845223Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.685068622Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.685762246Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.686467571Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:37.687117574Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:38.676121863Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48091"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:54:38.676228805Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"39da503a-6f4a-4b83-bbab-38fd24420797","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48091","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":31,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:39.513530022Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:39.514424647Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:39.515178032Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:39.515902867Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:39.516618312Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:39.517303587Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.681145179Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.683270973Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.683990498Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.684779724Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.68552649Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.686210534Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.686927389Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.687442999Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.68798381Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.688684375Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.689372849Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:47.690069104Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:49.518443108Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:49.519246725Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:49.520023501Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:49.520715605Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:49.52142591Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:49.522112854Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.684939476Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.686997329Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.687739564Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.688437069Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.689115683Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.689779426Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.69046912Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.690988491Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.691503142Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.692174866Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.69287408Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:57.693545254Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:59.522778817Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:59.523653404Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:59.52439651Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:59.525106014Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:59.525795249Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:54:59.526486063Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.359099358Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.359280182Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bd9e0c36-ed09-4f77-bd3e-b777dcd22e40","remote_addr":"[fd00:1122:3344:101::a]:58062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.374128068Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.374145428Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"35989db6-a3f7-4003-b9d6-e54ae1bd751d","remote_addr":"[fd00:1122:3344:101::a]:58062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.383296937Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"48895d6a-43e2-4330-ab34-e8c99d861452","remote_addr":"[fd00:1122:3344:101::a]:58062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.388445614Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.388631168Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"e27a609b-e202-4321-83b7-d600180a815a","remote_addr":"[fd00:1122:3344:101::c]:39708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.390257941Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39441"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.390364753Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8670ea8c-2386-4d88-9850-b1545425c6b8","remote_addr":"[fd00:1122:3344:101::b]:39441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.390557677Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.390664539Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"793e6ffe-a912-4d76-be82-28b61058c1ef","remote_addr":"[fd00:1122:3344:101::b]:42847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.405290592Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.405304282Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b2c2576a-1433-49ce-8255-ddfe80eeade2","remote_addr":"[fd00:1122:3344:101::b]:42847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":47,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.41442075Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e0373a70-68bc-44e3-a0d8-7b48be290f29","remote_addr":"[fd00:1122:3344:101::b]:42847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.441958939Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.442045181Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ddc511e3-2a5f-4c8f-b5d6-7aa91b94cd1e","remote_addr":"[fd00:1122:3344:101::c]:48280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.455423398Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.455561921Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"177414b3-4618-46bf-8357-c667209b0c4b","remote_addr":"[fd00:1122:3344:101::a]:55179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.457344268Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59044"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.45743673Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.503702906Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.569134828Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"3f29fbb4-bd67-456f-a0c8-8e412a238452","remote_addr":"[fd00:1122:3344:101::a]:59044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111722,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.569309871Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35344"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.569362092Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63685"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.569570097Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.569588667Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.569606988Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b23049e9-906e-4d0f-a567-5072557a868b","remote_addr":"[fd00:1122:3344:101::c]:48280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":216,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.574421447Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"33a4e482-b3b3-43cb-a4e4-a81c00475f3c","remote_addr":"[fd00:1122:3344:101::a]:59044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4850,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.579990592Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"06937b12-966b-4367-a910-83506766da47","remote_addr":"[fd00:1122:3344:101::c]:48280","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":223,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.61084359Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.672961433Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.67617723Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5ce5c92a-6b09-4822-9b51-5ed55d2b1a19","remote_addr":"[fd00:1122:3344:101::b]:35344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103364,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.677277083Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d14fa860-f4a9-4075-864f-c05ca1df0914","remote_addr":"[fd00:1122:3344:101::b]:35344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":678,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.710711314Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.772324087Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5e5eb983-222a-4b2c-acb5-bd9f76b98682","remote_addr":"[fd00:1122:3344:101::c]:63685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":202606,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:03.772725235Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a38693e2-9a88-4b85-8b24-c608bea7a20a","remote_addr":"[fd00:1122:3344:101::c]:63685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":157,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.688786134Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.690867607Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.691615882Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.692302977Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.693013631Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.693688155Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.694356059Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.69487389Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.69536957Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.696060144Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.696729198Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:07.697388072Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:08.710608161Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41809"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:08.710683482Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"926b1b83-401d-4837-a86b-5f3fdaaea9e0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41809","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":29,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:09.528170896Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:09.529053045Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:09.529812721Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:09.530568266Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:09.531308811Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:09.532001006Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.691960138Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.69396144Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.694633453Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.695294087Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.695988731Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.696624315Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.697265848Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.697748778Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.698259088Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.698939572Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.699597316Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:17.700239889Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:19.531673118Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:19.532615847Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:19.533370773Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:19.534108438Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:19.534837383Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:19.535567528Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.69453947Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.696651154Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.697373498Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.698087763Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.698778167Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.699460812Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.700138276Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.700648836Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.701191268Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.701887152Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.702557366Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:27.703227499Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:29.535948304Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:29.536855743Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:29.537744122Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:29.538510947Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:29.539245682Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:29.539962307Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.316569175Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.316758608Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"71fb6466-c816-4793-84ae-8540127dced2","remote_addr":"[fd00:1122:3344:101::a]:36444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.376769118Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:42909"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.376988803Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"9a50d4ab-448f-493f-ba60-67ca843ddf6f","remote_addr":"[fd00:1122:3344:101::c]:42909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.384031618Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.384051978Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7ee81e7e-5ba6-4c00-a903-c88d5ea13921","remote_addr":"[fd00:1122:3344:101::a]:36444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.397413514Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"96e072bd-04a6-485f-abf3-aeddc3a72403","remote_addr":"[fd00:1122:3344:101::a]:36444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":241,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.408584675Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.408607066Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ff6329de-a57d-428c-ab40-846e3bb75e46","remote_addr":"[fd00:1122:3344:101::c]:42909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.419133934Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b9351acb-b096-485f-a4ed-560ae56612d2","remote_addr":"[fd00:1122:3344:101::c]:42909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":263,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.548775663Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.548948536Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8415f5bb-c121-4044-aba3-f6c6795a14ef","remote_addr":"[fd00:1122:3344:101::c]:50960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.560644708Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.560792101Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8ec13a53-57b7-4e20-9755-623af6489c43","remote_addr":"[fd00:1122:3344:101::b]:53199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.574955384Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60072"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.575049376Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.621647739Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.690576463Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"98ada1dd-1794-4c7a-a42b-e9feb68e8554","remote_addr":"[fd00:1122:3344:101::c]:60072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":115570,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.69088159Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54031"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.691052843Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.691083634Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"48344523-1e3c-4a06-bf85-fe429b262b30","remote_addr":"[fd00:1122:3344:101::b]:53199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":194,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.691193206Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5d11ab68-1bb4-4c11-a715-67875601e3eb","remote_addr":"[fd00:1122:3344:101::c]:60072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":266,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.691261217Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"8587315e-8130-4da7-8d38-20aafdf0e020","remote_addr":"[fd00:1122:3344:101::a]:54031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":201,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.703769436Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:53524"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.703997601Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"e4b73485-29e4-4133-b603-b70eed329ad6","remote_addr":"[fd00:1122:3344:101::b]:53524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":172,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.707523654Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7dad9941-a2a0-47f0-b88a-c2f4b7d9f471","remote_addr":"[fd00:1122:3344:101::b]:53199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":207,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.722182596Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43560"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.722287548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.767186607Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.827482522Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f1005f28-44c4-43f0-be4e-4d5cbd4f02b7","remote_addr":"[fd00:1122:3344:101::a]:43560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105221,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.827714177Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:43427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.827929472Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cce62658-234a-4325-94f9-f0d8c6879a91","remote_addr":"[fd00:1122:3344:101::a]:43560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":209,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.828001793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.873295549Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.92896798Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"798f4aec-8cf7-4fc1-9588-1d42913896e3","remote_addr":"[fd00:1122:3344:101::b]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101065,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:33.929354218Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d560043b-cf92-473f-840b-0d97db741a11","remote_addr":"[fd00:1122:3344:101::b]:43427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.698380998Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.70040202Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.701121285Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.701799159Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.702486313Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.703227378Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.703922053Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.704445203Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.704987095Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.705650158Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.706314952Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:37.707019757Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:38.743614679Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:55:38.743719711Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ffeee9ea-a216-465d-a949-9261628332c5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45174","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":37,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:39.540522107Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:39.541309964Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:39.542011378Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:39.542775834Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:39.543439268Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:39.544116572Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.702216966Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.704280409Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.705013764Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.705717438Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.706455203Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.707134738Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.707809842Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.708333732Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.708858843Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.709547038Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.710299773Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:47.711020998Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:49.54458632Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:49.545425507Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:49.546165662Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:49.546879677Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:49.547598702Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:49.548291216Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.704852589Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.707036454Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.707763789Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.708459033Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.709215139Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.709937884Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.710621628Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.711155369Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.71170207Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.712383394Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.713100709Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:57.713783883Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:59.548697844Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:59.54951034Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:59.550226665Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:59.550918749Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:59.551585433Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:55:59.552342749Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.23091178Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38094"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.231064013Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a2072e03-f7f5-4f8b-aedf-a844f442cb09","remote_addr":"[fd00:1122:3344:101::c]:38094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.286931178Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.286965448Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"738a89d7-22c2-4b20-9376-a2443c15cc83","remote_addr":"[fd00:1122:3344:101::c]:38094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.297003306Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e272ef5f-b917-4c9d-be70-48ee8bec9610","remote_addr":"[fd00:1122:3344:101::c]:38094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.362072Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48688"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.362256994Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"0a838d22-9303-4461-9c2d-e7a4526947e2","remote_addr":"[fd00:1122:3344:101::c]:48688","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.402760561Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.402831932Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2602ba30-6ec4-47a7-a86b-6180a17b9a0c","remote_addr":"[fd00:1122:3344:101::b]:41368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.418968116Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.419011377Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"55851502-ff63-4c51-9c0e-b602e80dfd0b","remote_addr":"[fd00:1122:3344:101::b]:41368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.433163539Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55385"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.433282062Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.480350784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.542834766Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"45662d6d-2eeb-486a-a40e-d96bd7a0f3d6","remote_addr":"[fd00:1122:3344:101::c]:55385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109585,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.542991319Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36530"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.54306052Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:64664"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.543092701Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52447"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.543167762Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47360"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.543305615Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.543567841Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"43211651-9349-4a1a-a442-bcf352106809","remote_addr":"[fd00:1122:3344:101::c]:55385","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":422,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.543619142Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.54837638Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"83cf4611-c3ad-42cc-8ed8-fc2c125bb2ac","remote_addr":"[fd00:1122:3344:101::a]:36530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4971,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.548537644Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d1be8325-0995-43a7-afe0-343a82efb057","remote_addr":"[fd00:1122:3344:101::b]:41368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5252,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.548560834Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"73454678-8b51-4950-bdf6-2015c714f64f","remote_addr":"[fd00:1122:3344:101::b]:64664","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4996,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.548791269Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"de96eca9-d924-4ab8-8347-249279a4a34f","remote_addr":"[fd00:1122:3344:101::a]:47360","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":226,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.583499756Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.589064411Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.589124652Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"42fd4b63-4656-49ae-b43b-daad0c70c74d","remote_addr":"[fd00:1122:3344:101::a]:36530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":156,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.600418176Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"048e394b-3e2b-48a2-b588-3d51528e0c50","remote_addr":"[fd00:1122:3344:101::a]:36530","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":768,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.644378784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.644432005Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5fabdc58-d8b4-4edf-906b-1f7f97f777a0","remote_addr":"[fd00:1122:3344:101::a]:58158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100906,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.650710325Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a0689ce7-0368-4a24-ba34-648c7a82ba42","remote_addr":"[fd00:1122:3344:101::a]:58158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":768,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.685050035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.744606526Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"539333c1-e8eb-4aa9-b3ff-8214ae440a1a","remote_addr":"[fd00:1122:3344:101::b]:52447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":201027,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:03.745005074Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0ecbde03-ec30-4eae-834a-f0bbcc3d054e","remote_addr":"[fd00:1122:3344:101::b]:52447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":182,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.708938262Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.711055596Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.7117699Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.712441714Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.713113778Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.713785242Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.714445026Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.715005777Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.715542128Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.716178861Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.716859946Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:07.717510389Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:08.777677048Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34678"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:08.77776406Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"583eff48-a7ce-42b8-a239-4f1460e1209a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34678","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:09.555836909Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:09.556594945Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:09.55729562Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:09.557988844Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:09.558664308Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:09.559335071Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.71280186Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.714832152Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.715533847Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.716217031Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.716895995Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.717562079Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.718232553Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.718725503Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.719224693Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.719878187Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.72054763Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:17.721193454Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:19.560233319Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:19.561098866Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:19.561872993Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:19.562849723Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:19.563726601Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:19.564450166Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.717325242Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.719359534Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.720050149Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.720711422Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.721422217Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.722100111Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.722797355Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.723334466Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.723882038Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.724544091Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.725236316Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:27.725893529Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:29.563937844Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:29.564654249Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:29.565395214Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:29.566077078Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:29.566751872Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:29.567468677Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.338334085Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:54666"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.338651912Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7256fae6-e16d-47a7-95e9-2603ce6161ee","remote_addr":"[fd00:1122:3344:101::c]:54666","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.511118436Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65350"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.511211937Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"0f20673d-94e2-40b1-a6b2-2ced0022d809","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.580109632Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55337"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.580240494Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"db17296c-1f98-4c6b-89ed-cd768a2a9ebd","remote_addr":"[fd00:1122:3344:101::c]:55337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":56,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.595638362Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.595674343Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5bea8f5d-e788-4250-acff-873ce32117c3","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.607299123Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"02b4eef8-bede-4c68-9849-c2c94815dadb","remote_addr":"[fd00:1122:3344:101::a]:65350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":201,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.627089112Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40430"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.627190734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.673980821Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.746881078Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d3799add-b2bf-4965-b7fa-014c41274a34","remote_addr":"[fd00:1122:3344:101::c]:40430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":119734,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747219785Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747335117Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41716"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747419169Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:60467"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747506771Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33340"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747621593Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cf01f1f0-b51a-481a-b8f7-3b1099e06b42","remote_addr":"[fd00:1122:3344:101::c]:40430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":395,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747836238Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"52bfa820-bb83-4c3f-8ab0-b7707343298f","remote_addr":"[fd00:1122:3344:101::b]:47715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":179,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747854578Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747896289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.74794805Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.747988961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.748004001Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.755961196Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.756430785Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"6000dd2a-f1d1-4c13-9919-99bc1688e2d3","remote_addr":"[fd00:1122:3344:101::c]:55337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9227,"response_code":"204"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.756964546Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.76634004Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"ec2eccd7-17c5-4a65-820c-fafca9d896b7","remote_addr":"[fd00:1122:3344:101::c]:55337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":359,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.778633894Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60463"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.779103594Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"0df51da0-49f9-42dc-a932-bee982a05bb0","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":369,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.798342342Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.868346438Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"472cc0b4-e5ce-493b-bfcb-74fe9f168c37","remote_addr":"[fd00:1122:3344:101::a]:41716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":120551,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.868772037Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"b81bbb68-11a5-4036-b307-fcbd551893e7","remote_addr":"[fd00:1122:3344:101::a]:41716","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.87856852Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.878975188Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"38b19854-81bc-473a-baa8-473cbf5cdc6e","remote_addr":"[fd00:1122:3344:101::b]:36159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":196,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.881664293Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:44441"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.881767016Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.894925727Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52681"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.895149612Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:33.942968891Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008064286Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9905a179-3fb8-4b1e-80b5-f526338fa8ac","remote_addr":"[fd00:1122:3344:101::b]:52681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113004,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008359392Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37581"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008376292Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008389052Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"d8899fa9-ec6f-42af-be0e-1277e34e5826","remote_addr":"[fd00:1122:3344:101::b]:47715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":134,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008420283Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38084"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008502205Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:54256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008562346Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"9b11b9b5-bec6-4790-9a9a-f0202e7b1d72","remote_addr":"[fd00:1122:3344:101::b]:52681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":243,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008777331Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008833942Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008917323Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008932314Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008942424Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008976795Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.008987015Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009004185Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009047996Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"721d888b-9365-4474-ba1f-aec5e0c7b2ae","remote_addr":"[fd00:1122:3344:101::a]:37581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":458,"response_code":"204"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009056666Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009092467Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009100827Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009111858Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009122788Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009138978Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009152738Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: aa72dcff-155a-44f9-a2e7-012afe411f7a, kind: Service { id: 5cab3305-2cf8-4205-93bd-c0a5ec70688d }, name: Name(\"external-dns-5cab3305-2cf8-4205-93bd-c0a5ec70688d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 251])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e09fd754-5a2d-4ad5-af87-f65f4fe294cc, kind: Service { id: e2d2fc82-684b-4404-ae54-ce48e33d41e3 }, name: Name(\"external-dns-e2d2fc82-684b-4404-ae54-ce48e33d41e3\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 189, 54])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6605325d-9219-4c52-a246-627be8937804, kind: Service { id: c3da9028-2f6b-4199-8d06-98c01f75b9bf }, name: Name(\"nexus-c3da9028-2f6b-4199-8d06-98c01f75b9bf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 51])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: cd9c7f98-ca50-485a-b02a-a17a60c7c197, kind: Service { id: 0339f029-6c8d-4b23-83cc-6885965f777d }, name: Name(\"nexus-0339f029-6c8d-4b23-83cc-6885965f777d\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 179, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f30f824b-79e3-4862-bff7-ff49b8d24fe2, kind: Service { id: ea2b5fec-1bb6-4488-9cf9-d60d9b99619b }, name: Name(\"nexus-ea2b5fec-1bb6-4488-9cf9-d60d9b99619b\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 146, 126])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009164348Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"182d0bfa-c433-42b7-a910-75ed5fdb6ba2","remote_addr":"[fd00:1122:3344:101::c]:38084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":530,"response_code":"204"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.00921182Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.00926001Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009275941Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009285591Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009311982Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009322172Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.009353052Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1c63ada4-5c68-498b-8a10-82182456284f","remote_addr":"[fd00:1122:3344:101::b]:54256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":571,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.01262202Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"a2f8c0ef-b107-40dd-b172-a1770102958f","remote_addr":"[fd00:1122:3344:101::c]:60467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":264969,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.012652711Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.012693572Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.012898516Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.01988911Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"57105e40-739c-4e0f-9f0f-9c61fe41b789","remote_addr":"[fd00:1122:3344:101::b]:47715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":254,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.236323333Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"f2d4bb33-aad6-40a7-9b02-5093f1b8233c","remote_addr":"[fd00:1122:3344:101::a]:33340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":488501,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.236407795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.236422725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.236698001Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:34.470923721Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"f126b640-c0bb-4fc5-873b-566fca0db39f","remote_addr":"[fd00:1122:3344:101::b]:44441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":589151,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.721599969Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.72361585Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.724303105Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.725000639Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.725667593Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.726363357Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.727016401Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.727514151Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.728051682Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.728688585Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.729337479Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:37.729992412Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:38.812053934Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:56:38.812158776Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ccf56619-3315-464e-9958-2ecf6c624b00","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46104","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:39.56818893Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:39.568988257Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:39.569729882Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:39.570471267Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:39.571158331Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:39.571866066Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.724707602Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.726798115Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.72753727Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.728241984Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.728961439Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.729661254Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.730342138Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.730829248Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.731366719Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.732049543Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.732747498Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:47.733411841Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:49.572749093Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:49.573541999Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:49.574264654Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:49.574958978Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:49.575646173Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:49.576333047Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.729293525Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.731410749Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.732141924Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.732831198Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.733525622Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.734238087Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.734939361Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.735453452Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.735982963Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.736664567Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.737360982Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:57.738013605Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:59.577015309Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:59.577787035Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:59.57848584Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:59.579185214Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:59.57993155Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:56:59.580825808Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.231568485Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.231798769Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7c2a9ae3-e551-4148-93eb-800c89aa6da0","remote_addr":"[fd00:1122:3344:101::c]:51256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.25410052Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.254126211Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"76c2f876-c924-480c-8074-9ccd2c946c72","remote_addr":"[fd00:1122:3344:101::c]:51256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.265095257Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b818b190-3fec-463d-9acb-9431b6e8f688","remote_addr":"[fd00:1122:3344:101::c]:51256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":199,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.31358246Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.313703742Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1e485963-8c64-4366-b935-ebe1a8fdde29","remote_addr":"[fd00:1122:3344:101::b]:45939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.324521235Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50176"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.324635048Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.37994246Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.452623752Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0e86c861-2042-424b-abe4-74b1bb04f6fa","remote_addr":"[fd00:1122:3344:101::c]:50176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":128036,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.452804166Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44060"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.452834896Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:55810"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.452871227Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62504"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.45299834Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.453041581Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"86b31d7d-2ae7-42a2-9d2f-dc3fa9b62eaa","remote_addr":"[fd00:1122:3344:101::b]:45939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":173,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.453113892Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fc7adf25-13b1-4222-bfec-a5ad1a100691","remote_addr":"[fd00:1122:3344:101::a]:44060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":86,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.453269755Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d776ca88-b68e-4970-bac2-ad6fe2c4fb15","remote_addr":"[fd00:1122:3344:101::c]:50176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":346,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.453289006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.460004385Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"5da640d5-93a6-4175-9998-52d2319fce51","remote_addr":"[fd00:1122:3344:101::c]:55810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6753,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.460228979Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.460787941Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"6d61b446-61a3-4c45-a5a8-55c888337722","remote_addr":"[fd00:1122:3344:101::b]:57789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":326,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.469997331Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"79101e9a-89a7-48ac-a11d-b7305ee5912e","remote_addr":"[fd00:1122:3344:101::b]:45939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":215,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.477298982Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56603"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.495391196Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.495428977Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a005a31b-43f0-44ec-a229-d19805ebdcb8","remote_addr":"[fd00:1122:3344:101::a]:44060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":117,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.501824561Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.519246979Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.519621407Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"7fb898a1-5f4a-4d50-858e-fe63eb368ce0","remote_addr":"[fd00:1122:3344:101::a]:54014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":225,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.523099049Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"8a52e6d1-1246-4874-a4b3-9b7b0bf604f1","remote_addr":"[fd00:1122:3344:101::a]:44060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":243,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.566334462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.566391413Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a58bee2a-027d-4607-8c98-d59d90014a75","remote_addr":"[fd00:1122:3344:101::b]:62504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113216,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.570901826Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c788a6e5-6057-4acd-8450-5d7e8d66c776","remote_addr":"[fd00:1122:3344:101::b]:62504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":697,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.604598443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.663186354Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"40fb58eb-a35c-4b72-8a3d-d9a09551d828","remote_addr":"[fd00:1122:3344:101::a]:56603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":185744,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:03.663561661Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"23f87980-b62c-400d-a35a-244fbea3cbe8","remote_addr":"[fd00:1122:3344:101::a]:56603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.734006991Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.736049753Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.736729177Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.737421331Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.738110076Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.738778179Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.739434953Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.739942493Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.740434704Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.741104077Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.741757391Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:07.742403774Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:08.844396488Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41163"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:08.84448934Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"37c81e87-1a13-4a98-9907-e6eb70ac0854","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41163","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":35,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:09.581752206Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:09.582544212Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:09.583257687Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:09.583965822Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:09.584651236Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:09.585367821Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.737867129Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.739931702Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.740652107Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.741327541Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.742008765Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.742655208Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.743324752Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.743852673Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.744385294Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.745047607Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.745688451Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:17.746341054Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:19.587089205Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:19.587856098Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:19.588520152Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:19.5892809Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:19.589973004Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:19.590645798Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.741995593Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.744050835Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.744716969Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.745375172Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.746044156Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.74671747Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.747394744Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.747869404Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.748354144Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.748994797Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.749662021Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:27.750324965Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:29.590967573Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:29.59175715Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:29.592454224Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:29.593123558Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:29.593775881Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:29.594442135Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.291711423Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36210"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.291826705Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"e8def734-cf3c-45a6-b70f-7ab58f734565","remote_addr":"[fd00:1122:3344:101::c]:36210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":71,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.370896049Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.37094823Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"728ce95e-1ada-4d2e-933a-8a4f7c9e926e","remote_addr":"[fd00:1122:3344:101::c]:36210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":122,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.385632983Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"db431d81-a36f-4341-b268-33759db08e71","remote_addr":"[fd00:1122:3344:101::c]:36210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.493346419Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:62986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.493566594Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"5c800333-eb8f-4b90-ac9a-65651d34a956","remote_addr":"[fd00:1122:3344:101::c]:62986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.508545523Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33926"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.508627075Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.557779501Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.62305585Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"5d8441eb-9b83-4b68-8a71-a95091d1d240","remote_addr":"[fd00:1122:3344:101::c]:33926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":114463,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.623169912Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:47979"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.623215683Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.62356463Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"2537be36-23da-4729-9bd8-f1bb542aa58f","remote_addr":"[fd00:1122:3344:101::b]:61915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":244,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.623586221Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ff952e3c-4b57-4bac-a836-47bcd394be52","remote_addr":"[fd00:1122:3344:101::b]:47979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.623687523Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"85ca1634-dd4c-42c5-9155-2bff435e2bad","remote_addr":"[fd00:1122:3344:101::c]:33926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":259,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.632009125Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60057"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.632166638Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a215e1f1-737c-4211-9574-3e5e22c99575","remote_addr":"[fd00:1122:3344:101::a]:60057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.647489795Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.647507665Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8bee81ed-6134-49a9-a960-0fd713f96d2c","remote_addr":"[fd00:1122:3344:101::a]:60057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":76,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.674888991Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.674910641Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"108a64ee-1268-4ba4-b425-bf2d32ddd3bf","remote_addr":"[fd00:1122:3344:101::b]:47979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":50,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.685687414Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7898257e-f23b-4a87-930b-9c2743950a55","remote_addr":"[fd00:1122:3344:101::b]:47979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.70967215Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.709846763Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"9ba86f56-f728-4f49-a287-142eda967706","remote_addr":"[fd00:1122:3344:101::a]:51828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.715263185Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49625"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.715366767Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.759243094Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.828008785Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"05a5208e-06c8-4fef-b5ed-73e9eb9c0ed2","remote_addr":"[fd00:1122:3344:101::b]:49625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112674,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.828272291Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:35419"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.828535266Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0a262829-5280-4bf7-9a96-7f967290313e","remote_addr":"[fd00:1122:3344:101::b]:49625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":262,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.828546706Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6c5c03fa-90ae-4a0f-a532-3c1c5c42dabc","remote_addr":"[fd00:1122:3344:101::a]:60057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":370,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.828609948Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.872390063Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.932564676Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fbbb687e-bab5-47b2-8a87-3a8b76106736","remote_addr":"[fd00:1122:3344:101::a]:35419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:33.944812849Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"edf0299e-aecb-4d65-8605-2130768040b9","remote_addr":"[fd00:1122:3344:101::a]:35419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.745497604Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.747578447Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.748359903Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.749009236Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.74968141Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.750336213Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.751001237Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.751473107Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.751979067Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.752627211Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.753288265Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:37.753920838Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:38.877307343Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:57:38.877429456Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4e4572b3-607d-4da0-8ffe-19f4f6af75d0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60224","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:39.595572717Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:39.596376884Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:39.597127559Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:39.597855934Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:39.598532808Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:39.599218692Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.749448994Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.75216916Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.752884555Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.753574409Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.754271463Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.754941137Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.755612261Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.756117831Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.756657183Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.757312506Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.75798011Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:47.758656374Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:49.600239862Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:49.60113373Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:49.601883776Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:49.60257965Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:49.603252644Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:49.603920208Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.753275231Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.755573859Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.756270493Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.756951837Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.757640212Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.758317675Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.758993069Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.759538641Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.760050321Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.760723805Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.76144104Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:57.762090184Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:59.604235433Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:59.60507833Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:59.605783415Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:59.606468849Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:59.607136363Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:57:59.607780746Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.221892995Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.222034688Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"14a9159d-cf9c-4b34-92d5-7e3f1de7e584","remote_addr":"[fd00:1122:3344:101::c]:64234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.232197938Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50694"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.232436793Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"07a5def1-5374-4607-bcf8-0c2bbfdd6313","remote_addr":"[fd00:1122:3344:101::c]:50694","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":158,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.238374446Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.238390796Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1bec86b1-939a-4699-8aee-9723e3cf251f","remote_addr":"[fd00:1122:3344:101::c]:64234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.247525515Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"30dbb9b9-5703-4c82-a654-f6cfdb72beac","remote_addr":"[fd00:1122:3344:101::c]:64234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":159,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.264167729Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.264237011Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"94db6cbc-0147-4105-9f20-b7763218e7c3","remote_addr":"[fd00:1122:3344:101::a]:58266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.313760744Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.313776244Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b93956e5-2be3-48a0-9783-1ba17585f294","remote_addr":"[fd00:1122:3344:101::a]:58266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":57,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.316192104Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.316285706Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bbf9aeae-cec8-4b4a-a768-a94ac262c40d","remote_addr":"[fd00:1122:3344:101::b]:37508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.333004972Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.333051073Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"9ac5542e-c5e1-42e0-8aa1-71b0662468ea","remote_addr":"[fd00:1122:3344:101::b]:37508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.340150299Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45544"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.340266092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.392787307Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.45879016Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bf82febb-1f25-4f81-9997-c2025fd7a918","remote_addr":"[fd00:1122:3344:101::c]:45544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":118552,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.459103137Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53137"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.459197279Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51814"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.459606587Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0e1266bd-be28-4062-848f-6d3d54ea6762","remote_addr":"[fd00:1122:3344:101::c]:45544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":359,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.459685329Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"1571c483-f229-4243-b009-3e080e0619e3","remote_addr":"[fd00:1122:3344:101::b]:37508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":500,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.45971652Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"c1d44d41-3b9f-4314-91b3-196ba1b0bd2e","remote_addr":"[fd00:1122:3344:101::b]:51814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":316,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.459782681Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"77d4a5e2-b4c0-40ac-870b-0204f53a48ef","remote_addr":"[fd00:1122:3344:101::a]:58266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":593,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.459826422Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"df5d167a-2c65-455f-a1cc-4bad90d732e6","remote_addr":"[fd00:1122:3344:101::a]:53137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":332,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.497227265Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62474"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.497346647Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.538997528Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.601234374Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b04d6250-46b3-4c7a-a72e-9b17fc3a4add","remote_addr":"[fd00:1122:3344:101::a]:62474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103941,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.601347577Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45629"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.601544301Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.60538151Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a5f92431-9eb5-4cdc-a75c-0650a0b75041","remote_addr":"[fd00:1122:3344:101::a]:62474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3821,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.63975092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.698596356Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cec39677-80f3-4678-a9f9-9ea2b6a2eaa0","remote_addr":"[fd00:1122:3344:101::b]:45629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97126,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:03.699022925Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7d58fb01-ffc4-4c9e-b38b-6625eba6ee17","remote_addr":"[fd00:1122:3344:101::b]:45629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.75669305Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.758839325Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.759518049Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.760234644Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.760912567Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.761603482Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.762261375Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.762745785Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.763283976Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.76394184Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.764649055Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:07.765314878Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:08.91172469Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61474"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:08.911822512Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0b1fa9c7-1f07-4bbc-a4fb-51577ff2640c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61474","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":29,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:09.608360397Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:09.609129342Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:09.609827167Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:09.610535642Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:09.611224766Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:09.61191488Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.760853045Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.762948158Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.763660763Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.764402038Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.765261656Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.765981201Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.766679005Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.767182035Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.767694896Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.76835636Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.769047614Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:17.769708138Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:19.612886408Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:19.613699415Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:19.61440198Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:19.615088374Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:19.615768458Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:19.616456242Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.764651281Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.766698464Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.767401328Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.768081582Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.769341608Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.770017492Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.770678566Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.771177546Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.771688507Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.77235468Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.773010604Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:27.773632465Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:29.6169441Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:29.617728387Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:29.618445591Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:29.619156686Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:29.61985014Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:29.620553045Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.563735002Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60118"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.563990768Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ad18e341-3ee2-4b95-b7f4-3cadd0156b03","remote_addr":"[fd00:1122:3344:101::a]:60118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":172,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.565852626Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(1603470) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.565891177Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"PUT","req_id":"9169b335-0c60-46a4-adbc-5fb2a96ac425","remote_addr":"[fd00:1122:3344:101::a]:60118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1725,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.725508446Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53953"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.736089734Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 8ca6dc86-4894-40ad-89a9-2ffb8afbe6bd, project_id: 5ad6fbb0-2a15-4329-9a4a-4ec092cd06bb }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-09T02:58:31.725276121Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(1603470))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-1810ccebbbd\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 74f9670a-f051-45c9-b80a-537ec20a74c6, read_only_parent: Some(Volume { block_size: 512, id: 5ec32d83-2dfa-4600-a1e8-2bee2a8cbe40, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 80e9fd42-8ab3-4b12-962e-f591d358eb46, key: Some(\"5QDKeLeVPKWx75O1V2fyaJFJKIxnL1WnP2JWMbRUu5w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 74f9670a-f051-45c9-b80a-537ec20a74c6, key: Some(\"bw9wKpITbdjZylyq0imovR1gduUFdKPwtmEYQZ03nrQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::12]:19003\", \"[fd00:1122:3344:101::1a]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-1810ccebbbd\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.736152386Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.736609395Z","hostname":"buskin","pid":1572,"propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(1603470))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2945403d-7dac-4915-b39b-33737fbebf32, kind: Instance { id: 80b85a0b-333c-468c-9cee-5d2eba219497 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(1603470), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-1810ccebbbd\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 74f9670a-f051-45c9-b80a-537ec20a74c6, read_only_parent: Some(Volume { block_size: 512, id: 5ec32d83-2dfa-4600-a1e8-2bee2a8cbe40, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 80e9fd42-8ab3-4b12-962e-f591d358eb46, key: Some(\"5QDKeLeVPKWx75O1V2fyaJFJKIxnL1WnP2JWMbRUu5w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 74f9670a-f051-45c9-b80a-537ec20a74c6, key: Some(\"bw9wKpITbdjZylyq0imovR1gduUFdKPwtmEYQZ03nrQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::12]:19003\", \"[fd00:1122:3344:101::1a]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-1810ccebbbd\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-09T02:58:31.725276121Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.738123846Z","hostname":"buskin","pid":1572,"uri":"/vmms/cb313128-bef6-4891-a142-8c709c04545b","method":"PUT","req_id":"58c62600-4fec-44f4-be8f-023c99dd2220","remote_addr":"[fd00:1122:3344:101::a]:53953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":12535,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.927615203Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.927835097Z","hostname":"buskin","pid":1572,"uri":"/vmms/cb313128-bef6-4891-a142-8c709c04545b/state","method":"PUT","req_id":"78e380f6-1a80-4ff1-b79b-0f76a05e9285","remote_addr":"[fd00:1122:3344:101::a]:45852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":123,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.931476752Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 1603470 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.932234528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:31.978665518Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:32.005432681Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:32.032182953Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:65345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:32.032361887Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6e18a208-2f8f-4784-b7d0-f3e44dcd7c54","remote_addr":"[fd00:1122:3344:101::a]:65345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:32.080535623Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {2945403d-7dac-4915-b39b-33737fbebf32: {10.151.1.176: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}, 10.151.1.175: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}}} }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:32.462058787Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.343574055Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:64251"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.416783697Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44282"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.427989249Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.428239084Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"77694e02-9706-44cc-887c-39e6c3898c34","remote_addr":"[fd00:1122:3344:101::b]:58344","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.480692148Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36348"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.48082083Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"31eb2a02-7796-4360-bac7-5efe7682b8fc","remote_addr":"[fd00:1122:3344:101::c]:36348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.519688764Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49249"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.541590477Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {2945403d-7dac-4915-b39b-33737fbebf32: {10.151.1.175: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}, 10.151.1.176: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.541628137Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4abd016-d9c5-414b-8631-aa9ea025f61a","remote_addr":"[fd00:1122:3344:101::c]:36348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.544525947Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45732"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.544812543Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"e30e78d3-b859-404a-8be8-9379b21ff144","remote_addr":"[fd00:1122:3344:101::c]:45732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":195,"response_code":"200"} {"msg":"Profile for oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.599681377Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.599934592Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.611513991Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:46142"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.611823758Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.640263456Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"07d0ce69-c7e0-4461-84ca-9aad9d955e70","remote_addr":"[fd00:1122:3344:101::c]:36348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":635,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.659950403Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:48092"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.660149757Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bfa04d3f-2621-4400-8d7e-74e8325d4fa7","remote_addr":"[fd00:1122:3344:101::b]:48092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.662844302Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.713772975Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {2945403d-7dac-4915-b39b-33737fbebf32: {10.151.1.175: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}, 10.151.1.176: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.713837086Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"fe321ec3-1504-4b0c-b50b-0584cc01ee18","remote_addr":"[fd00:1122:3344:101::b]:48092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":236,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.741235363Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3ce0c5db-959f-4de8-8bf5-fc06594696d9","remote_addr":"[fd00:1122:3344:101::b]:48092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1109,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.744673903Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56560"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.753450075Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"72fb9e33-0d86-460d-999a-b6a14028671f","remote_addr":"[fd00:1122:3344:101::c]:46142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141734,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.753469395Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.761394529Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3591d9c9-30cd-44a8-8d5c-74baaffd51b9","remote_addr":"[fd00:1122:3344:101::c]:46142","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1002,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.809618136Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 71.671625ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:33.84516809Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.862628051Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.863073001Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"68844734-30f1-4c17-b6a6-c04f9c376192","remote_addr":"[fd00:1122:3344:101::a]:53159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":270,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.882678906Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"a9150440-757e-46f5-a1fd-23fa65d9ba4f","remote_addr":"[fd00:1122:3344:101::b]:56560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":137940,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:33.883044483Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"94d15d35-ab95-49d9-b852-60551c053613","remote_addr":"[fd00:1122:3344:101::b]:56560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":127,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 129.198368ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:33.940412409Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:34.002256587Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:50947"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:34.002578773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4081"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:34.069614379Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:34.170682198Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"32bebbf0-13cc-4501-915b-d2cc4567389f","remote_addr":"[fd00:1122:3344:101::a]:50947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":168226,"response_code":"204"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 206.371078ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:34.189444515Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:34.190051868Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"56c06da2-b590-47a8-a782-d1892e9612e1","remote_addr":"[fd00:1122:3344:101::a]:50947","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":323,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 386.906406ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:34.413126537Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 1.056946488s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:34.815971593Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 740.348309ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:35.885349362Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 524.785487ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:36.639782003Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 1.324349757s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.17866954Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.770280866Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.772332518Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.773014823Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.773678666Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.77435898Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.775013984Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.775665007Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.776184998Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.776692899Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.777343622Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.778023506Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:37.77869225Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"wait for service Some(\"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\") failed: Property not found. retry in 1.257106876s","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:38.518762234Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:38.944920307Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:38.947241589Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"0f72c777-f606-41f7-9045-87a4ecaab62e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54684","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:39.619985942Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:39.620773928Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:39.621483672Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:39.622163767Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:39.622838291Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:39.623510354Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"Started propolis in zone: oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.808551829Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:1787"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.820300761Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:1796"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.855705923Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:1820"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.992760225Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.993600463Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.993613633Z","hostname":"buskin","pid":1572,"uri":"/vmms/cb313128-bef6-4891-a142-8c709c04545b/state","method":"GET","req_id":"7a3555f4-8009-4e7b-ba03-8b2dedfb672e","remote_addr":"[fd00:1122:3344:101::b]:49249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6473794,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.993660704Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"457d160a-9d5a-456f-acf5-2a0ba1b6dc8a","remote_addr":"[fd00:1122:3344:101::a]:65345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7913197,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.993724045Z","hostname":"buskin","pid":1572,"uri":"/vmms/cb313128-bef6-4891-a142-8c709c04545b/state","method":"GET","req_id":"a7216f49-6589-4817-b29d-afea57c87de1","remote_addr":"[fd00:1122:3344:101::a]:44282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6576799,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.993745646Z","hostname":"buskin","pid":1572,"uri":"/vmms/cb313128-bef6-4891-a142-8c709c04545b/state","method":"GET","req_id":"97dd0bc5-fc52-4bda-af8a-7ad22067d02a","remote_addr":"[fd00:1122:3344:101::c]:64251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6649978,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-11-09T02:58:39.994768387Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.99537772Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.99539502Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-09T02:58:39.994768387Z }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:39.99540392Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-09T02:58:39.994768387Z }, migration_in: None, migration_out: None }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-11-09T02:58:40.023993151Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.024026282Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.024055032Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-09T02:58:40.023993151Z }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.024071743Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-09T02:58:40.023993151Z }, migration_in: None, migration_out: None }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.080205833Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:469","new_vers":"RouterVersion { router_id: 2bc1c013-f3c1-4db1-bb74-56e3984db9d8, version: 3 }","old_vers":"RouterVersion { router_id: 2bc1c013-f3c1-4db1-bb74-56e3984db9d8, version: 3 }","subnet":"RouterId { vni: Vni(1603470), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.080305195Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6ce85993-656a-45f6-80f4-9f0cb96a3c97","remote_addr":"[fd00:1122:3344:101::a]:65345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":261,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.120967285Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:60900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.121049097Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"195d57d0-3231-4b13-9dbb-c419aa199970","remote_addr":"[fd00:1122:3344:101::a]:60900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":42,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.163117566Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {2945403d-7dac-4915-b39b-33737fbebf32: {10.151.1.176: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}, 10.151.1.175: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.163137927Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1d467410-fd31-4300-91f3-6d238044aff0","remote_addr":"[fd00:1122:3344:101::a]:60900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:40.17880026Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d6e03c03-afa3-4bca-b2e0-c5c383ac47c2","remote_addr":"[fd00:1122:3344:101::a]:60900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":213,"response_code":"204"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.773595983Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.775704356Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.77639946Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.777041055Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.777778329Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.778433142Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.779090616Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.779579226Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.780095557Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.78073971Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.781403724Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:47.782039607Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.47725084Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58878"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.477869013Z","hostname":"buskin","pid":1572,"uri":"/vmms/cb313128-bef6-4891-a142-8c709c04545b/state","method":"PUT","req_id":"93dc57f3-9dc4-414d-8ba4-abd8c96b6a0a","remote_addr":"[fd00:1122:3344:101::a]:58878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":536,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-11-09T02:58:49.477974515Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.478000345Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.478017346Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-09T02:58:49.477974515Z }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.478029966Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-09T02:58:49.477974515Z }, migration_in: None, migration_out: None }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-11-09T02:58:49.488672596Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.488688526Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.488708407Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-11-09T02:58:49.488672596Z }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.488734087Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.488754208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.488970472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:49.489560044Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/290b5ea3-1da3-4d8a-a79d-8837f0b3c303/debug/bundle/zone\", \"/pool/int/dd41f58e-e0a0-4e88-9b00-217e5d2f507b/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/31d0192b-e979-447e-9118-b71dd165c143/crypt/debug\", \"/pool/ext/99589249-e847-4d9d-a74c-b1f313d5d545/crypt/debug\", \"/pool/ext/74b698d6-a94b-4491-b5c6-a2bac3974e86/crypt/debug\", \"/pool/ext/62a6cce5-1929-4693-88be-42b0a605236c/crypt/debug\", \"/pool/ext/8b88214b-96a1-4c48-a234-7cd78b00e420/crypt/debug\", \"/pool/ext/db69e33d-bffe-47ae-b048-940c028eae57/crypt/debug\", \"/pool/ext/83c1e528-b687-4cd0-a6ee-e747f81e14fc/crypt/debug\", \"/pool/ext/736f1ad3-6d18-406b-b576-5193fe8077c2/crypt/debug\", \"/pool/ext/0c46804c-6ce7-415e-af40-2b1d35c7620b/crypt/debug\"] }","zone_name":"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:49.625008014Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:49.625838081Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:49.626615377Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:49.627310071Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:49.628014056Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:49.62871365Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:50.153601018Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b\", bundle_id: 0eed4309-1700-4504-beaa-1a480eba0c33 }, time_created: 2024-11-09T02:58:49.490546385Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_cb313128-bef6-4891-a142-8c709c04545b","v":0,"name":"SledAgent","level":40,"time":"2024-11-09T02:58:50.153659719Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:51.945409467Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:51.965845269Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(1603470), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:51.96588958Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(1603470), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:51.966434221Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-09T02:58:51.966399581Z }, migration_in: None, migration_out: None }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.006938108Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-09T02:58:51.966399581Z }, migration_in: None, migration_out: None }","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.009887999Z","hostname":"buskin","pid":1572,"instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","propolis_id":"cb313128-bef6-4891-a142-8c709c04545b","instance_id":"80b85a0b-333c-468c-9cee-5d2eba219497","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.605229372Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34514"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.605331154Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"6e713802-1f07-4707-848d-452526ff200c","remote_addr":"[fd00:1122:3344:101::b]:34514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.619083409Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {2945403d-7dac-4915-b39b-33737fbebf32: {10.151.1.176: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}, 10.151.1.175: {0df7e5b9-3539-48e3-9b34-00c5061bdee6}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.619108589Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c3cb5a66-6643-4c81-9cad-a7d6972b5fba","remote_addr":"[fd00:1122:3344:101::b]:34514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.627915311Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"948404d7-d2b1-4703-8727-29a872a47fc9","remote_addr":"[fd00:1122:3344:101::b]:34514","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.739214641Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.739419826Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1024fd78-c18f-4093-99b4-af3587ac9737","remote_addr":"[fd00:1122:3344:101::b]:51010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.899136317Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:40316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:52.89929728Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1006da1b-ac5f-4ddd-953d-9db8bfa9ad2b","remote_addr":"[fd00:1122:3344:101::a]:40316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.001226116Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.001325058Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d4710fe6-e787-467d-bea5-0dd6977d3bfe","remote_addr":"[fd00:1122:3344:101::a]:63055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.023486956Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.023562668Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1233"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.023568318Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"81662459-8272-48e8-9b0a-93e2472e8dfc","remote_addr":"[fd00:1122:3344:101::a]:63055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":167,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.031281317Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"14d10e1e-6c77-4ef7-8500-972d88df3650","remote_addr":"[fd00:1122:3344:101::a]:63055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.78386081Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.783989723Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"1535e819-510b-4e5e-8a80-a8d38c43baec","remote_addr":"[fd00:1122:3344:101::a]:54718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":103,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.799308169Z","hostname":"buskin","pid":1572,"sled_id":"f032ba4a-ac68-41c0-a068-00d3371df648","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1207","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.799326989Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cb6dabd0-966c-44bb-8112-79369df5217e","remote_addr":"[fd00:1122:3344:101::a]:54718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-09T02:58:53.807551179Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"7679e1d0-0bdf-4d5f-a104-2206eb02e300","remote_addr":"[fd00:1122:3344:101::a]:54718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":195,"response_code":"204"}